diff --git a/.project-docs/30-worklog/tasks/20260813-go-remaining-modules-7d3a9e42.md b/.project-docs/30-worklog/tasks/20260813-go-remaining-modules-7d3a9e42.md index 1ef1d6f..4ba85ca 100644 --- a/.project-docs/30-worklog/tasks/20260813-go-remaining-modules-7d3a9e42.md +++ b/.project-docs/30-worklog/tasks/20260813-go-remaining-modules-7d3a9e42.md @@ -8,7 +8,7 @@ - Worktree: /Users/brother7/Documents/AI/NianAIGC-go-remaining-7d3a9e42 - Base commit: d0207fcebe6ea4fb3ba80dce8c012b3c2170de40 - Owner: codex -- Status: Planning +- Status: Ready for Integration ## Scope @@ -51,11 +51,35 @@ ## Outcome -- Not completed. +- Completed in the feature worktree. Implemented and composed the remaining Go + modules: administration and organization lifecycle, assets and hardened + storage/remote fetch, billing ledger/catalog/settlement, usage reporting, + jobs/providers/webhooks/worker loop, templates/prompt, settings/logging, + public and compatibility HTTP routes, localstore adapters, PostgreSQL + adapters, and lifecycle fencing migration 0002. Added language-neutral + contracts and executable TypeScript/Go consumers while preserving the + existing Next production ownership and deployment topology. + +- Added final lifecycle hardening for streamed HTTP event logging, unique + worker lease tokens and CAS fencing, commit-outcome reconciliation, retry + recovery handles, readiness checks for lifecycle columns, mock output + registration, usage filter options, public webhook/priority validation, and + a single settings/billing-account source. ## Verification -- Not run. +- `CGO_ENABLED=0 go test -count=1 ./...` — PASS +- `CGO_ENABLED=0 go test -race -count=1 ./internal/...` (all migrated + packages) — PASS +- `npm run go:vet` — PASS +- `npm run go:build` — PASS +- `npm test -- --run` — 57 files / 172 tests PASS +- `npx tsc --noEmit --incremental false` — PASS +- `npm run build` — PASS (Next build) +- `node scripts/check-ack-manifests.mjs` and `npm run deploy:check` — PASS +- `git diff --check`, `gofmt` cleanliness, and production boundary diff + checks — PASS +- Independent `sol_reviewer` final review — PASS; no blocking findings. ## Follow-ups @@ -63,7 +87,11 @@ real application role and verified-CA TLS before production cutover. - Validate OSS compatibility, provider credentials, external Webhooks, Worker drain/recovery, and rollout/rollback against production-like infrastructure. +- Keep Go unrouted until the explicit production cutover review; do not delete + the Next handlers or change Docker/ACK/Ingress ownership in this task. ## Promotion Candidates -- None recorded. +- Preserve the new Go module map and contract fixtures as candidates for the + next serialized Integration Gate to promote into canonical architecture + memory. This feature task does not modify shared canonical documents. diff --git a/backend/README.md b/backend/README.md index f639067..b4ecd18 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,19 +1,31 @@ -# Go backend foundation +# Go modular backend -This directory contains the first implementation slice of ADR-003. It is a -runnable compatibility foundation, but it is **not** the current production API +This directory contains the separately runnable Go implementation of ADR-003. +It now owns the checked-in compatibility implementation for all 66 explicit +HTTP method/path entries, but it is **not** the current production traffic owner: Next.js, the Node Worker, Docker Compose, and the ACK manifests remain -unchanged until later route-by-route cutover work passes the shared contracts. +unchanged until a later, explicit cutover. Implemented Modules: -- `identity`: legacy `zhinian_session` HMAC/chunking, database-refreshed - authorization, and the password-login lifecycle. +- `identity` and `administration`: legacy `zhinian_session` HMAC/chunking, + database-refreshed authorization, password lifecycle, accounts, and + organizations. +- `assets`: scoped registration, upload, download, deletion, local filesystem, + bounded remote import, and Alibaba Cloud OSS adapters. +- `jobs`, `providers`, and `orchestration`: provider preparation/protocols, + idempotent creation, embedded WorkerLoop, retries, output assets, usage, + refunds, Seedance settlement, and signed Webhooks. +- `billing` and `usage`: integer-fen quote/wallet/ledger/catalog behavior and + tenant-scoped reports. +- `templates`, `prompt`, `settings`, and `logging`: the remaining compatibility + modules used by the HTTP surface. - `postgres`: fail-closed configuration, verified-CA TLS, readiness, atomic - password lockout transactions, and calls to the existing claim and wallet - PostgreSQL functions. -- `httpapi`: process health, database readiness, current-session, password - login, and logout handlers. + account mutations, and calls to the existing claim and wallet PostgreSQL + functions. PostgreSQL is the production relational source of truth. +- `localstore`: a mutex-protected, non-durable, single-process development + store covering the same business Module ports. +- `httpapi`: the complete checked-in route compatibility surface. - `application`: composition and the `cmd/zhinian-api` process entry point. From the repository root: @@ -33,11 +45,12 @@ server, use a different port: ZHINIAN_DATA_BACKEND=local GO_BACKEND_PORT=8080 ./backend/zhinian-api ``` -`/api/health`, `/api/ready`, `/api/auth/me`, `/api/auth/password`, and -`/api/auth/logout` are implemented in the separately runnable Go process. No -Ingress, Docker, ACK, Secret, or Worker ownership has moved to Go yet, so -Next.js remains the production owner of every route. +In local mode, persistent business data is process-local and is discarded on +restart; it is intended only for development and contract smoke tests. The +default demo identity is the same optional-auth super administrator used by the +current Next development flow. -The authentication handlers reuse the shared Cookie contracts and PostgreSQL -Adapters. Self-service/admin password mutation and production route ownership -remain with Next.js until later path-level cutover. +No Ingress, Docker, ACK, Secret, or Worker ownership has moved to Go yet, so +Next.js remains the deployed owner of every route. Real RDS/CA, OSS, provider, +Webhook, Worker drain/recovery, and rollback validation are mandatory before +that route ownership changes. diff --git a/backend/cmd/zhinian-api/main.go b/backend/cmd/zhinian-api/main.go index 0732b04..0d2b8e8 100644 --- a/backend/cmd/zhinian-api/main.go +++ b/backend/cmd/zhinian-api/main.go @@ -14,6 +14,7 @@ import ( "time" "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/application" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/settings" ) const shutdownTimeout = 10 * time.Second @@ -28,14 +29,18 @@ func run() error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - app, err := application.New(application.Options{Context: ctx}) + getenv, err := runtimeGetenv(os.Getenv) + if err != nil { + return fmt.Errorf("load Go backend runtime settings: %w", err) + } + app, err := application.New(application.Options{Context: ctx, Getenv: getenv}) if err != nil { return fmt.Errorf("initialize Go backend: %w", err) } defer app.Close() server := &http.Server{ - Addr: listenAddress(os.Getenv), + Addr: listenAddress(getenv), Handler: app.Handler(), ReadHeaderTimeout: 5 * time.Second, } @@ -66,6 +71,29 @@ func run() error { return nil } +func runtimeGetenv(process func(string) string) (func(string) string, error) { + path := strings.TrimSpace(process("ZHINIAN_SETTINGS_FILE")) + if path == "" { + path = ".env.local" + } + values := map[string]string{} + for _, key := range settings.RuntimeSettingKeys() { + if value := process(key); value != "" { + values[key] = value + } + } + merged, err := settings.LoadEnvironment(path, values) + if err != nil { + return nil, err + } + return func(name string) string { + if value, exists := merged[name]; exists { + return value + } + return process(name) + }, nil +} + func listenAddress(getenv func(string) string) string { host := strings.TrimSpace(getenv("GO_BACKEND_HOST")) if host == "" { diff --git a/backend/cmd/zhinian-api/main_test.go b/backend/cmd/zhinian-api/main_test.go index 9a6c853..0777a97 100644 --- a/backend/cmd/zhinian-api/main_test.go +++ b/backend/cmd/zhinian-api/main_test.go @@ -1,6 +1,10 @@ package main -import "testing" +import ( + "os" + "path/filepath" + "testing" +) func TestListenAddressDefaultsToLoopbackAndSupportsExplicitBinding(t *testing.T) { if got := listenAddress(func(string) string { return "" }); got != "127.0.0.1:8080" { @@ -11,3 +15,18 @@ func TestListenAddressDefaultsToLoopbackAndSupportsExplicitBinding(t *testing.T) t.Fatalf("listenAddress(explicit) = %q", got) } } + +func TestRuntimeGetenvLoadsWhitelistedSettingsAndPreservesProcessPrecedence(t *testing.T) { + path := filepath.Join(t.TempDir(), ".env.local") + if err := os.WriteFile(path, []byte("IMAGE_GENERATE_ENGINE=evolink\nEVOLINK_API_KEY=file-secret\nDATABASE_URL=postgres://must-not-load\n"), 0o600); err != nil { + t.Fatal(err) + } + process := map[string]string{"ZHINIAN_SETTINGS_FILE": path, "EVOLINK_API_KEY": "process-secret", "ZHINIAN_DATA_BACKEND": "local"} + getenv, err := runtimeGetenv(func(name string) string { return process[name] }) + if err != nil { + t.Fatal(err) + } + if getenv("IMAGE_GENERATE_ENGINE") != "evolink" || getenv("EVOLINK_API_KEY") != "process-secret" || getenv("DATABASE_URL") != "" || getenv("ZHINIAN_DATA_BACKEND") != "local" { + t.Fatalf("loaded engine=%q key=%q database=%q backend=%q", getenv("IMAGE_GENERATE_ENGINE"), getenv("EVOLINK_API_KEY"), getenv("DATABASE_URL"), getenv("ZHINIAN_DATA_BACKEND")) + } +} diff --git a/backend/internal/administration/model.go b/backend/internal/administration/model.go index b3834eb..4efd611 100644 --- a/backend/internal/administration/model.go +++ b/backend/internal/administration/model.go @@ -141,6 +141,30 @@ type UpdateAccountInput struct { Password *string ClearLoginLock bool } + +// AccountUpdate is the storage intent for an account PATCH. Implementations +// must only mutate fields whose pointers/flags are present. In particular, +// security state that is not part of this intent must remain database-owned so +// a concurrent password change or login attempt cannot be overwritten by a +// stale account snapshot. +type AccountUpdate struct { + Actor Actor + DisplayName *string + Role *Role + OrganizationID *string + Status *Status + PasswordHash *PasswordHash + ClearLoginLock bool + IncrementSessionVersion bool + UpdatedAt time.Time +} + +// AtomicAccountUpdater is an optional deep storage seam for implementations +// that can apply AccountUpdate atomically. Store remains backward compatible +// for process-local/test adapters; durable stores should implement this seam. +type AtomicAccountUpdater interface { + ApplyAccountUpdate(context.Context, string, AccountUpdate) (Account, error) +} type UpdateOrganizationInput struct { Name *string Status *Status diff --git a/backend/internal/administration/service.go b/backend/internal/administration/service.go index 296c9dd..05c494a 100644 --- a/backend/internal/administration/service.go +++ b/backend/internal/administration/service.go @@ -109,6 +109,7 @@ func (s *Service) UpdateAccount(ctx context.Context, actor Actor, id string, pat return Account{}, problem(ErrorForbidden, "组织管理员不能修改账号角色或归属。") } next := current + update := AccountUpdate{Actor: actor} mutates := false if patch.DisplayName != nil { name := strings.TrimSpace(*patch.DisplayName) @@ -116,16 +117,20 @@ func (s *Service) UpdateAccount(ctx context.Context, actor Actor, id string, pat return Account{}, problem(ErrorValidation, "显示名称不能为空。") } next.DisplayName = name + update.DisplayName = &name } if patch.Role != nil { if !validRole(*patch.Role) { return Account{}, problem(ErrorValidation, "账号角色不正确。") } next.Role = *patch.Role + update.Role = patch.Role mutates = true } if patch.OrganizationID != nil { next.OrganizationID = strings.TrimSpace(*patch.OrganizationID) + organizationID := next.OrganizationID + update.OrganizationID = &organizationID mutates = true } if patch.Status != nil { @@ -133,6 +138,7 @@ func (s *Service) UpdateAccount(ctx context.Context, actor Actor, id string, pat return Account{}, problem(ErrorValidation, "账号状态不正确。") } next.Status = *patch.Status + update.Status = patch.Status mutates = true } if err := s.validateMembership(ctx, next.Role, next.OrganizationID); err != nil { @@ -147,15 +153,23 @@ func (s *Service) UpdateAccount(ctx context.Context, actor Actor, id string, pat return Account{}, infrastructure("hash password", err) } next.PasswordHash, next.PasswordSalt = hashed.Hash, hashed.Salt + update.PasswordHash = &hashed mutates = true } if patch.ClearLoginLock { next.FailedLoginCount, next.LockedUntil = 0, nil + update.ClearLoginLock = true } if mutates { next.SessionVersion++ + update.IncrementSessionVersion = true } next.UpdatedAt = s.now() + update.UpdatedAt = next.UpdatedAt + if atomic, ok := s.store.(AtomicAccountUpdater); ok { + updated, err := atomic.ApplyAccountUpdate(ctx, id, update) + return updated, infrastructure("update account", err) + } updated, err := s.store.UpdateAccount(ctx, next) return updated, infrastructure("update account", err) } diff --git a/backend/internal/administration/service_test.go b/backend/internal/administration/service_test.go index b99c2cf..c95ee32 100644 --- a/backend/internal/administration/service_test.go +++ b/backend/internal/administration/service_test.go @@ -217,3 +217,54 @@ func (s *fakeStore) DeleteOrganization(context.Context, string) error { return s func (s *fakeStore) CountOrganizationMembers(context.Context, string) (int, error) { return s.memberCount, s.err } + +func TestUpdateAccountPrefersAtomicIntentOverStaleFullRowWrite(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + base := &fakeStore{ + accounts: map[string]Account{"user-1": { + ID: "user-1", DisplayName: "Before", Role: RoleUser, OrganizationID: "org-1", Status: StatusActive, + PasswordHash: "stale-hash", PasswordSalt: "stale-salt", SessionVersion: 4, + }}, + organizations: map[string]Organization{"org-1": {ID: "org-1", Status: StatusActive}}, + } + store := &atomicAccountStore{fakeStore: base, result: Account{ + ID: "user-1", DisplayName: "After", Role: RoleUser, OrganizationID: "org-1", Status: StatusActive, + PasswordHash: "concurrent-password-hash", PasswordSalt: "concurrent-password-salt", SessionVersion: 8, + }} + service := NewService(store, WithClock(func() time.Time { return now })) + name := "After" + + got, err := service.UpdateAccount(context.Background(), Actor{ID: "super", Role: RoleSuperAdmin}, "user-1", UpdateAccountInput{DisplayName: &name}) + if err != nil { + t.Fatal(err) + } + if store.fullRowWrites != 0 { + t.Fatalf("full-row writes=%d, want zero", store.fullRowWrites) + } + if store.id != "user-1" || store.update.DisplayName == nil || *store.update.DisplayName != "After" { + t.Fatalf("atomic update=(id=%q, update=%#v)", store.id, store.update) + } + if store.update.PasswordHash != nil || store.update.IncrementSessionVersion { + t.Fatalf("display-only update must not overwrite password or increment session: %#v", store.update) + } + if got.PasswordHash != "concurrent-password-hash" || got.SessionVersion != 8 { + t.Fatalf("result=%#v, want database-current security state", got) + } +} + +type atomicAccountStore struct { + *fakeStore + id string + update AccountUpdate + result Account + fullRowWrites int +} + +func (s *atomicAccountStore) UpdateAccount(ctx context.Context, account Account) (Account, error) { + s.fullRowWrites++ + return s.fakeStore.UpdateAccount(ctx, account) +} +func (s *atomicAccountStore) ApplyAccountUpdate(_ context.Context, id string, update AccountUpdate) (Account, error) { + s.id, s.update = id, update + return s.result, nil +} diff --git a/backend/internal/application/application.go b/backend/internal/application/application.go index 09c1a42..6b4df49 100644 --- a/backend/internal/application/application.go +++ b/backend/internal/application/application.go @@ -5,10 +5,24 @@ import ( "context" "net/http" "os" + "path/filepath" + "strings" + "time" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/httpapi" "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/localstore" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/orchestration" "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/postgres" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/prompt" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/publicapi" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/webhook" ) type Options struct { @@ -22,11 +36,23 @@ type Options struct { // CredentialAuthenticator is the narrow Password Login persistence seam. // Production defaults to the same PostgreSQL Store used for authorization. CredentialAuthenticator identity.CredentialAuthenticator + // BlobStore and RemoteFetcher support storage-specific integration tests and + // alternate deployments. Production defaults to the hardened local store; + // a fully configured OSS environment is composed below. + BlobStore assets.BlobStore + RemoteFetcher assets.RemoteFetcher + // ProviderRegistry can replace all external adapters in deterministic tests. + ProviderRegistry jobs.ProviderRegistry + // Log adapters remain injectable for deterministic composition tests. + // Runtime settings intentionally have one concrete source so /api/settings + // and billing account endpoints cannot observe different stores. + Logs httpapi.LogService } type App struct { handler http.Handler db *postgres.Module + worker *jobs.WorkerLoop } func New(options Options) (*App, error) { @@ -62,14 +88,56 @@ func New(options Options) (*App, error) { } }() readiness := databaseReadiness{config: config, store: database.Store} + + // PostgreSQL remains the production source of truth. Local mode swaps every + // business persistence port as one coherent process-local unit so modules do + // not accidentally call the nil-backed PostgreSQL shell used for readiness. + var authorizationStore identity.AuthorizationSnapshotLoader = database.Store + var credentialStore identity.CredentialAuthenticator = database.Store + var passwordChangeStore identity.PasswordChanger = database.Store + var administrationStore administration.Store = database.Store + var assetCatalog assets.Catalog = database.Store + var billingStore billing.Store = database.Store + var walletPoster billing.WalletPoster = postgres.NewBillingWalletPoster(database.Store) + var usageRepository usage.Repository = postgres.NewUsageRepository(database.Store) + var templateCatalog templates.Catalog = database.Store + var jobStore jobs.Store = database.Store + var creationState orchestration.CreationStateWriter = database.Store + var jobState orchestration.JobStateWriter = database.Store + var settlementState orchestration.SettlementStateWriter = database.Store + if config.Backend == postgres.BackendLocal { + store := localstore.New() + authorizationStore = store + credentialStore = store + passwordChangeStore = store + administrationStore = store + assetCatalog = store + billingStore = store + walletPoster = store + usageRepository = store + templateCatalog = store + jobStore = store + creationState = store + jobState = store + settlementState = store + } var resolver httpapi.SessionResolver if authConfig.Configured { loader := options.AuthorizationLoader if loader == nil { - loader = database.Store + loader = authorizationStore } resolver = identity.NewResolver(loader, authConfig.SessionSecret, "platform", nil) } + authState := httpapi.AuthState{Required: authConfig.Required, Configured: authConfig.Configured} + platformAuthorizer, err := httpapi.NewPlatformAuthorizer( + authState, + resolver, + httpapi.WithLocalDevelopmentFallback(config.Backend == postgres.BackendLocal && !strings.EqualFold(strings.TrimSpace(getenv("NODE_ENV")), "production")), + ) + if err != nil { + return nil, err + } authMe, err := httpapi.NewAuthMeHandler(httpapi.AuthState{ Required: authConfig.Required, Configured: authConfig.Configured, }, resolver) @@ -80,7 +148,7 @@ func New(options Options) (*App, error) { if authConfig.Configured { authenticator := options.CredentialAuthenticator if authenticator == nil { - authenticator = database.Store + authenticator = credentialStore } passwordIssuer = identity.NewPasswordLogin(authenticator, nil) } @@ -99,16 +167,189 @@ func New(options Options) (*App, error) { CookieSecure: cookieSecure, PublicBaseURL: publicBaseURL, }) - foundation := httpapi.NewHandler(readiness) + authCompatibility := httpapi.NewAuthCompatibilityHandler() + + publicAuthenticator := publicapi.NewAuthenticator(publicapi.Config{ + APIKeys: getenv("ZHINIAN_API_KEYS"), + InternalWorkerToken: getenv("ZHINIAN_INTERNAL_WORKER_TOKEN"), + Production: strings.EqualFold(strings.TrimSpace(getenv("NODE_ENV")), "production"), + }) + + administrationService := administration.NewService(administrationStore) + adminHandler, err := httpapi.NewAdminHandler(platformAuthorizer, administrationService) + if err != nil { + return nil, err + } + var passwordChangeHandler http.Handler = unavailableHandler(http.StatusServiceUnavailable) + if authConfig.Configured { + passwordChanger := identity.NewPasswordChange(passwordChangeStore, nil) + passwordChangeHandler, err = httpapi.NewAuthPasswordChangeHandler(httpapi.PasswordChangeConfig{ + SessionSecret: authConfig.SessionSecret, CookieSecure: cookieSecure, PublicBaseURL: publicBaseURL, + }, platformAuthorizer, passwordChanger) + if err != nil { + return nil, err + } + } + + blobStore := options.BlobStore + if blobStore == nil { + var configured bool + blobStore, configured, err = configuredOSSBlobStore(getenv) + if err != nil { + return nil, err + } + if !configured { + runtimeDirectory := strings.TrimSpace(getenv("ZHINIAN_RUNTIME_DIR")) + if runtimeDirectory == "" { + runtimeDirectory = filepath.Join(".runtime") + } + blobStore, err = assets.NewLocalFS(runtimeDirectory, firstNonEmpty(publicBaseURL, "http://127.0.0.1:3000")) + if err != nil { + return nil, err + } + } + } + remoteFetcher := options.RemoteFetcher + if remoteFetcher == nil { + remoteFetcher, err = assets.NewPublicHTTPRemoteFetcher( + 30*time.Second, + remoteAssetMaxBytes(getenv), + assets.NewPublicDestinationPolicy(nil, nil), + ) + if err != nil { + return nil, err + } + } + assetService := assets.NewService(assetCatalog, blobStore, remoteFetcher, nil, nil) + assetsHandler, err := httpapi.NewAssetsHandler(assetService, platformAuthorizer, publicAuthenticator, httpapi.AssetsConfig{ + MaxJSONBytes: positiveInt64Env(getenv, "ZHINIAN_MAX_JSON_BYTES", 1<<20), MaxUploadBytes: positiveInt64Env(getenv, "ZHINIAN_MAX_UPLOAD_BYTES", 20<<20), + }) + if err != nil { + return nil, err + } + + billingService := billing.NewService(billingStore, nil).SetEnabled(strings.TrimSpace(getenv("ZHINIAN_BILLING_REQUIRED")) != "0") + runtimeSettings := defaultSettingsService(getenv) + billingAccounts := settingsBillingAccountStore{service: runtimeSettings} + templateService := templates.NewService(templateCatalog, nil, nil) + logService := options.Logs + if logService == nil { + logService = defaultLogService(getenv) + } + var eventLogger EventLogger + if candidate, ok := logService.(EventLogger); ok { + eventLogger = candidate + } + miscHandler, err := httpapi.NewMiscHandler(httpapi.MiscDependencies{ + Platform: platformAuthorizer, Templates: templateService, PromptAssembler: prompt.Assemble, + Settings: runtimeSettings, Logs: logService, Public: publicAuthenticator, + Capabilities: capabilitySummary(getenv), PublicOrigin: publicBaseURL, + }) + if err != nil { + return nil, err + } + + providerRegistry := options.ProviderRegistry + if providerRegistry == nil { + providerRegistry = buildProviderRegistry(getenv) + } + jobService := jobs.NewService(jobStore, nil) + jobBuilder := jobs.ProviderJobBuilder{ + ImageProvider: imageProvider(getenv), VideoProvider: videoProvider(getenv), + ImageModel: imageModel(getenv), VideoModel: videoModel(getenv), ImageEngine: imageEngine(getenv), VideoEngine: videoEngine(getenv), + ImageEngines: providerImageTargets(getenv), VideoEngines: providerVideoTargets(getenv), NewID: applicationJobID, + } + usageService := usage.Service{ + Repository: usageRepository, + OrganizationOptions: usage.OrganizationOptionSourceFunc(func(ctx context.Context, requester usage.Requester) ([]usage.Option, error) { + organizations, listErr := administrationService.ListOrganizations(ctx, administration.Actor{ + ID: requester.AccountID, Role: administration.Role(requester.Role), OrganizationID: requester.OrganizationID, + }) + if listErr != nil { + return nil, listErr + } + options := make([]usage.Option, len(organizations)) + for index, organization := range organizations { + options[index] = usage.Option{Value: organization.ID, Label: organization.Name} + } + return options, nil + }), + } + usageHandler := httpapi.NewUsageHandler(platformAuthorizer, usageService, nil) + billingHandler := httpapi.NewBillingHandlerWithBuilder(platformAuthorizer, billingService, billingAccounts, jobBuilder) + ledger := billing.Ledger{Poster: walletPoster, NewID: func() string { return applicationID("ledger") }} + creation := orchestration.NewCreationCoordinator(jobBuilder, jobService, billingService, ledger, creationState) + refunds := orchestration.NewTerminalRefund(ledger, jobState) + usageRecorder := orchestration.NewUsageRecorder(usageService, func() string { return applicationID("usage") }, nil) + webhookSender, err := defaultWebhookSender(getenv) + if err != nil { + return nil, err + } + webhookBridge := orchestration.NewWebhookBridge(webhook.NewDeliverer(webhookSender, getenv("ZHINIAN_WEBHOOK_SECRET"), nil)) + outputs := orchestration.NewAssetOutputRegistrar(assetService, orchestration.ResolveProviderOutputURLs) + providerProcessor := jobs.ProviderProcessor{Providers: providerRegistry, Store: jobStore} + settlementProcessor := orchestration.NewSettlementProcessor(providerProcessor, ledger, settlementState, nil) + processor := orchestration.NewOutputRegisteringProcessor(settlementProcessor, outputs, jobState) + artifacts := orchestration.NewAssetArtifacts(assetService) + worker := jobs.NewWorker(jobStore, processor, refunds, usageRecorder, webhookBridge, jobs.WorkerConfig{ + BatchSize: int(positiveInt64Env(getenv, "ZHINIAN_WORKER_BATCH_SIZE", 3)), + LockTimeoutSeconds: int(positiveInt64Env(getenv, "ZHINIAN_WORKER_LOCK_TIMEOUT_SECONDS", 300)), + PollInterval: durationEnv(getenv, "ZHINIAN_WORKER_POLL_INTERVAL_MS", 5*time.Second), + }, nil) + jobsHandler, err := httpapi.NewJobsHandler(httpapi.JobsDependencies{ + Service: jobService, Platform: platformAuthorizer, Public: publicAuthenticator, + Builder: httpapi.ProviderBuilderAdapter{Builder: jobBuilder}, Creation: creation, Refunds: refunds, Artifacts: artifacts, Worker: WithTickEventLogging(worker, eventLogger), + }, httpapi.JobsConfig{MaxJSONBytes: positiveInt64Env(getenv, "ZHINIAN_MAX_JSON_BYTES", 1<<20), NewID: applicationJobID}) + if err != nil { + return nil, err + } + var workerLoop *jobs.WorkerLoop + if parseBool(getenv("ZHINIAN_GO_EMBEDDED_WORKER")) { + workerLoop = jobs.NewWorkerLoop(WithTickEventLogging(worker, eventLogger), jobs.LoopConfig{ + Interval: durationEnv(getenv, "ZHINIAN_WORKER_POLL_INTERVAL_MS", 5*time.Second), WorkerID: firstNonEmpty(getenv("ZHINIAN_WORKER_ID"), "embedded-worker"), + }) + workerLoop.Start(ctx) + } + + foundation := httpapi.NewHandler(readiness, httpapi.WithHealthDetails(runtimeHealthDetails(getenv))) mux := http.NewServeMux() mux.Handle("/api/auth/me", authMe) mux.Handle("/api/auth/password", authPassword) mux.Handle("/api/auth/logout", authLogout) + mux.Handle("/api/auth/password/change", passwordChangeHandler) + mux.Handle("/api/auth/login", authCompatibility) + mux.Handle("/api/auth/callback", authCompatibility) + mux.Handle("/api/auth/captcha", authCompatibility) + mux.Handle("/api/admin/accounts", adminHandler) + mux.Handle("/api/admin/accounts/", adminHandler) + mux.Handle("/api/admin/organizations", adminHandler) + mux.Handle("/api/assets", assetsHandler) + mux.Handle("/api/assets/", assetsHandler) + mux.Handle("/api/v1/assets", assetsHandler) + mux.Handle("/api/v1/assets/", assetsHandler) + mux.Handle("/uploads/", assetsHandler) + mux.Handle("/generated-results/", assetsHandler) + mux.Handle("/api/billing", billingHandler) + mux.Handle("/api/billing/", billingHandler) + mux.Handle("/api/admin/billing", billingHandler) + mux.Handle("/api/admin/billing/", billingHandler) + mux.Handle("/api/usage", usageHandler) + mux.Handle("/api/admin/usage", usageHandler) + mux.Handle("/api/generations/", jobsHandler) + mux.Handle("/api/v1/jobs", jobsHandler) + mux.Handle("/api/v1/jobs/", jobsHandler) + mux.Handle("/api/internal/worker/tick", jobsHandler) + mux.Handle("/api/image-templates", miscHandler) + mux.Handle("/api/image-templates/", miscHandler) + mux.Handle("/api/prompt/assemble", miscHandler) + mux.Handle("/api/settings", miscHandler) + mux.Handle("/api/logs", miscHandler) + mux.Handle("/api/v1/capabilities", miscHandler) + mux.Handle("/api/v1/openapi.json", miscHandler) mux.Handle("/", foundation) closeOnError = false return &App{ - db: database, - handler: mux, + db: database, handler: WithHTTPEventLogging(httpapi.WithRouteMethodCompatibility(mux), eventLogger), worker: workerLoop, }, nil } @@ -117,6 +358,9 @@ func (app *App) Handler() http.Handler { } func (app *App) Close() { + if app.worker != nil { + app.worker.Stop() + } if app.db != nil { app.db.Close() } diff --git a/backend/internal/application/application_test.go b/backend/internal/application/application_test.go index a424694..f41c8d4 100644 --- a/backend/internal/application/application_test.go +++ b/backend/internal/application/application_test.go @@ -11,7 +11,10 @@ import ( "time" "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/application" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/httpapi" "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/providers" ) func TestLocalApplicationServesFoundationHealthAndReadiness(t *testing.T) { @@ -72,6 +75,51 @@ func TestApplicationRejectsInvalidProductionDatabaseConfiguration(t *testing.T) } } +func TestProductionLocalBackendNeverGrantsAnonymousAdministrator(t *testing.T) { + app, err := application.New(application.Options{Getenv: applicationEnv(map[string]string{ + "NODE_ENV": "production", + "ZHINIAN_DATA_BACKEND": "local", + "ZHINIAN_AUTH_DISABLED": "true", + })}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + t.Cleanup(app.Close) + + response := httptest.NewRecorder() + app.Handler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/settings", nil)) + if response.Code != http.StatusUnauthorized { + t.Fatalf("status=%d body=%s, want 401", response.Code, response.Body.String()) + } +} + +func TestApplicationDerivesNextCompatibleMethodMatrixBeforeAuthentication(t *testing.T) { + app, err := application.New(application.Options{Getenv: applicationEnv(map[string]string{ + "NODE_ENV": "production", + "ZHINIAN_DATA_BACKEND": "local", + })}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(app.Close) + + for _, test := range []struct { + method, path, allow string + status int + }{ + {http.MethodOptions, "/api/admin/accounts", "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT", http.StatusNoContent}, + {http.MethodOptions, "/api/health", "GET, HEAD, OPTIONS", http.StatusNoContent}, + {http.MethodHead, "/api/health", "", http.StatusOK}, + {http.MethodPost, "/api/health", "", http.StatusMethodNotAllowed}, + } { + response := httptest.NewRecorder() + app.Handler().ServeHTTP(response, httptest.NewRequest(test.method, test.path, nil)) + if response.Code != test.status || response.Body.Len() != 0 || response.Header().Get("Allow") != test.allow { + t.Fatalf("%s %s status=%d allow=%q body=%q", test.method, test.path, response.Code, response.Header().Get("Allow"), response.Body.String()) + } + } +} + func TestApplicationServesAnonymousCurrentSessionWithAuthConfigurationState(t *testing.T) { tests := []struct { name string @@ -220,7 +268,7 @@ func TestApplicationComposesSignedCookieResolverWithAuthorizationLoader(t *testi } } -func TestApplicationUsesDatabaseAuthorizationAdapterByDefault(t *testing.T) { +func TestLocalApplicationUsesCoherentAuthorizationAdapterByDefault(t *testing.T) { secret := "local-default-adapter-secret-with-enough-entropy" app, err := application.New(application.Options{Getenv: applicationEnv(map[string]string{ "ZHINIAN_DATA_BACKEND": "local", @@ -235,7 +283,7 @@ func TestApplicationUsesDatabaseAuthorizationAdapterByDefault(t *testing.T) { Version: 1, AuthMode: identity.AuthModeUser, IssuedAt: time.Now().Add(-time.Minute).Unix(), ExpiresAt: time.Now().Add(time.Hour).Unix(), User: identity.User{ - ID: "user-1", Subject: "user-1", DisplayName: "User", ClientID: "platform", + ID: "demo-merchant", Subject: "demo-merchant", DisplayName: "Forged", ClientID: "platform", Authorities: []string{}, Scope: []string{}, }, } @@ -253,8 +301,121 @@ func TestApplicationUsesDatabaseAuthorizationAdapterByDefault(t *testing.T) { app.Handler().ServeHTTP(response, request) - if response.Code != http.StatusInternalServerError || response.Body.Len() != 0 { - t.Fatalf("response = %d %q, want empty 500 from unavailable local authorization adapter", response.Code, response.Body.String()) + if response.Code != http.StatusOK { + t.Fatalf("response = %d %q, want authenticated local session", response.Code, response.Body.String()) + } + var payload struct { + Authenticated bool `json:"authenticated"` + User struct { + ID, DisplayName, Role, OrganizationID string + } `json:"user"` + } + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + if !payload.Authenticated || payload.User.ID != "demo-merchant" || payload.User.DisplayName != "智念用户" || payload.User.Role != "super_admin" || payload.User.OrganizationID != "org-demo" { + t.Fatalf("payload = %#v", payload) + } +} + +func TestLocalApplicationBusinessModulesDoNotUseUnavailablePostgresShell(t *testing.T) { + app, err := application.New(application.Options{Getenv: applicationEnv(map[string]string{ + "ZHINIAN_DATA_BACKEND": "local", + "ZHINIAN_BILLING_REQUIRED": "0", + })}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + t.Cleanup(app.Close) + + for _, test := range []struct { + method, path, body string + want int + }{ + {http.MethodGet, "/api/assets", "", http.StatusOK}, + {http.MethodGet, "/api/image-templates", "", http.StatusOK}, + {http.MethodGet, "/api/usage", "", http.StatusOK}, + {http.MethodPost, "/api/generations/image", `{"prompt":"local mock"}`, http.StatusAccepted}, + } { + req := httptest.NewRequest(test.method, test.path, strings.NewReader(test.body)) + if test.body != "" { + req.Header.Set("Content-Type", "application/json") + } + res := httptest.NewRecorder() + app.Handler().ServeHTTP(res, req) + if res.Code != test.want { + t.Fatalf("%s %s = %d %q, want %d", test.method, test.path, res.Code, res.Body.String(), test.want) + } + } +} + +func TestLocalApplicationMockJobReachesSucceededWithStoredOutput(t *testing.T) { + app, err := application.New(application.Options{ + Getenv: applicationEnv(map[string]string{ + "ZHINIAN_DATA_BACKEND": "local", "ZHINIAN_BILLING_REQUIRED": "0", + "ZHINIAN_INTERNAL_WORKER_TOKEN": "worker-secret", + "ZHINIAN_WORKER_POLL_INTERVAL_MS": "1", + }), + ProviderRegistry: jobs.ProviderRegistry{ + "volcengine-visual": providers.NewMock("local-e2e"), + "evolink": providers.NewMock("local-e2e"), + "bailian": providers.NewMock("local-e2e"), + "seedance": providers.NewMock("local-e2e"), + "mock": providers.NewMock("local-e2e"), + }, + }) + if err != nil { + t.Fatalf("New() error = %v", err) + } + t.Cleanup(app.Close) + + create := httptest.NewRequest(http.MethodPost, "/api/generations/image", strings.NewReader(`{"prompt":"local output"}`)) + create.Header.Set("Content-Type", "application/json") + created := httptest.NewRecorder() + app.Handler().ServeHTTP(created, create) + if created.Code != http.StatusAccepted { + t.Fatalf("create = %d %q", created.Code, created.Body.String()) + } + var creation struct { + Job jobs.Job `json:"job"` + } + if err := json.NewDecoder(created.Body).Decode(&creation); err != nil || creation.Job.ID == "" { + t.Fatalf("creation = %#v, %v", creation, err) + } + + for tick := 0; tick < 2; tick++ { + if tick > 0 { + time.Sleep(2 * time.Millisecond) + } + request := httptest.NewRequest(http.MethodPost, "/api/internal/worker/tick", strings.NewReader(`{}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Authorization", "Bearer worker-secret") + response := httptest.NewRecorder() + app.Handler().ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("tick %d = %d %q", tick, response.Code, response.Body.String()) + } + } + + got := httptest.NewRecorder() + app.Handler().ServeHTTP(got, httptest.NewRequest(http.MethodGet, "/api/generations/image/"+creation.Job.ID, nil)) + if got.Code != http.StatusOK { + t.Fatalf("get = %d %q", got.Code, got.Body.String()) + } + var result struct { + Job jobs.Job `json:"job"` + } + if err := json.NewDecoder(got.Body).Decode(&result); err != nil { + t.Fatal(err) + } + if result.Job.Status != jobs.StatusSucceeded || len(result.Job.OutputAssetIDs) == 0 { + t.Fatalf("job = %#v", result.Job) + } + + assetsResponse := httptest.NewRecorder() + app.Handler().ServeHTTP(assetsResponse, httptest.NewRequest(http.MethodGet, "/api/assets", nil)) + if assetsResponse.Code != http.StatusOK || !strings.Contains(assetsResponse.Body.String(), result.Job.OutputAssetIDs[0]) { + t.Fatalf("assets = %d %q", assetsResponse.Code, assetsResponse.Body.String()) } } @@ -349,6 +510,34 @@ func TestApplicationLeavesLogoutAvailableWhenPasswordAuthenticationIsUnconfigure } } +func TestApplicationOwnsEveryCheckedInHTTPRoute(t *testing.T) { + app, err := application.New(application.Options{Getenv: applicationEnv(map[string]string{ + "ZHINIAN_DATA_BACKEND": "local", + "NODE_ENV": "production", + })}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + t.Cleanup(app.Close) + + for _, route := range httpapi.GoRouteSurface() { + route := route + t.Run(route.Method+" "+route.Path, func(t *testing.T) { + path := strings.ReplaceAll(route.Path, "{id}", "contract-id") + path = strings.ReplaceAll(path, "{path...}", "contract/file.png") + request := httptest.NewRequest(route.Method, "https://app.example.test"+path, strings.NewReader(`{}`)) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + + app.Handler().ServeHTTP(response, request) + + if response.Code == http.StatusNotFound { + t.Fatalf("route %s %s fell through to 404", route.Method, path) + } + }) + } +} + type applicationAuthorizationLoader struct { snapshot identity.AuthorizationSnapshot found bool diff --git a/backend/internal/application/event_logging.go b/backend/internal/application/event_logging.go new file mode 100644 index 0000000..1656391 --- /dev/null +++ b/backend/internal/application/event_logging.go @@ -0,0 +1,158 @@ +package application + +import ( + "context" + "io" + "net/http" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/logging" +) + +// EventLogger is the application-owned write seam implemented by +// *logging.Service. Keeping the seam here lets runtime composition provide a +// different sink without coupling either HTTP handlers or jobs to log storage. +type EventLogger interface { + Append(context.Context, logging.Input) (logging.Entry, error) +} + +// WithHTTPEventLogging records server failures while deliberately limiting +// request metadata to the method and URL path. Headers, cookies, query values, +// and bodies never enter the event. +func WithHTTPEventLogging(next http.Handler, logger EventLogger) http.Handler { + if next == nil { + next = http.NotFoundHandler() + } + return http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + writer := &eventResponseWriter{ResponseWriter: w, status: http.StatusOK} + defer func() { + if recovered := recover(); recovered != nil { + // Once a response has been streamed, HTTP cannot safely rewrite it. + // Before the first byte, however, we can still return a stable generic + // response without exposing the panic value. + if !writer.wroteHeader { + clearResponseHeaders(writer.Header()) + http.Error(writer, "Internal Server Error", http.StatusInternalServerError) + } + appendEvent(request.Context(), logger, logging.Input{ + Level: logging.Error, Source: "http", Message: "HTTP handler panic", + Status: http.StatusInternalServerError, Method: request.Method, Path: request.URL.Path, + }) + if writer.wroteHeader && writer.status != http.StatusInternalServerError { + // The only safe response after bytes have escaped is to abort the + // connection. net/http deliberately suppresses logging for this + // sentinel while preventing a truncated response from being reused. + panic(http.ErrAbortHandler) + } + return + } + if writer.status >= http.StatusInternalServerError { + appendEvent(request.Context(), logger, logging.Input{ + Level: logging.Error, Source: "http", Message: "HTTP request failed", + Status: writer.status, Method: request.Method, Path: request.URL.Path, + }) + } + }() + next.ServeHTTP(writer, request) + }) +} + +type eventResponseWriter struct { + http.ResponseWriter + status int + wroteHeader bool +} + +func (writer *eventResponseWriter) WriteHeader(status int) { + if writer.wroteHeader { + return + } + writer.status = status + writer.wroteHeader = true + writer.ResponseWriter.WriteHeader(status) +} + +func (writer *eventResponseWriter) Write(body []byte) (int, error) { + if !writer.wroteHeader { + writer.WriteHeader(http.StatusOK) + } + return writer.ResponseWriter.Write(body) +} + +// Unwrap lets http.ResponseController discover capabilities provided by the +// original server writer without making event logging own those interfaces. +func (writer *eventResponseWriter) Unwrap() http.ResponseWriter { + return writer.ResponseWriter +} + +// ReadFrom preserves io.Copy's streaming fast path for downloads while still +// recording the implicit 200 response status. +func (writer *eventResponseWriter) ReadFrom(source io.Reader) (int64, error) { + if !writer.wroteHeader { + writer.WriteHeader(http.StatusOK) + } + if readerFrom, ok := writer.ResponseWriter.(io.ReaderFrom); ok { + return readerFrom.ReadFrom(source) + } + return io.Copy(writer.ResponseWriter, source) +} + +func clearResponseHeaders(header http.Header) { + for name := range header { + delete(header, name) + } +} + +type eventLoggingTickRunner struct { + next jobs.TickRunner + logger EventLogger +} + +// WithTickEventLogging records both whole-tick failures and per-job failures. +// Error strings and worker/job inputs are intentionally omitted from events. +func WithTickEventLogging(next jobs.TickRunner, logger EventLogger) eventLoggingTickRunner { + return eventLoggingTickRunner{next: next, logger: logger} +} + +func (runner eventLoggingTickRunner) Tick(ctx context.Context, workerID string) (jobs.TickResult, error) { + return runner.record(ctx, func() (jobs.TickResult, error) { return runner.next.Tick(ctx, workerID) }) +} + +// TickLimit preserves the internal Worker HTTP seam while sharing the same +// best-effort event policy as the embedded loop. +func (runner eventLoggingTickRunner) TickLimit(ctx context.Context, workerID string, limit int) (jobs.TickResult, error) { + limited, ok := runner.next.(interface { + TickLimit(context.Context, string, int) (jobs.TickResult, error) + }) + if !ok { + return runner.Tick(ctx, workerID) + } + return runner.record(ctx, func() (jobs.TickResult, error) { return limited.TickLimit(ctx, workerID, limit) }) +} + +func (runner eventLoggingTickRunner) record(ctx context.Context, tick func() (jobs.TickResult, error)) (jobs.TickResult, error) { + result, err := tick() + if err != nil { + appendEvent(ctx, runner.logger, logging.Input{ + Level: logging.Error, Source: "worker", Message: "Worker tick failed", + }) + return result, err + } + for _, item := range result.Jobs { + if item.Error == "" { + continue + } + appendEvent(ctx, runner.logger, logging.Input{ + Level: logging.Error, Source: "worker", Message: "Worker job failed", + }) + } + return result, nil +} + +func appendEvent(ctx context.Context, logger EventLogger, input logging.Input) { + if logger == nil { + return + } + defer func() { _ = recover() }() + _, _ = logger.Append(context.WithoutCancel(ctx), input) +} diff --git a/backend/internal/application/event_logging_test.go b/backend/internal/application/event_logging_test.go new file mode 100644 index 0000000..5f3f9e0 --- /dev/null +++ b/backend/internal/application/event_logging_test.go @@ -0,0 +1,239 @@ +package application + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/logging" +) + +type recordingEventLogger struct { + mu sync.Mutex + inputs []logging.Input +} + +type panickingEventLogger struct{} + +func (panickingEventLogger) Append(context.Context, logging.Input) (logging.Entry, error) { + panic("log sink unavailable") +} + +func (logger *recordingEventLogger) Append(_ context.Context, input logging.Input) (logging.Entry, error) { + logger.mu.Lock() + defer logger.mu.Unlock() + logger.inputs = append(logger.inputs, input) + return logging.Entry{}, nil +} + +func (logger *recordingEventLogger) snapshot() []logging.Input { + logger.mu.Lock() + defer logger.mu.Unlock() + return append([]logging.Input(nil), logger.inputs...) +} + +func TestHTTPEventLoggingRecordsServerErrorsButNotClientErrors(t *testing.T) { + logger := &recordingEventLogger{} + handler := WithHTTPEventLogging(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/client-error" { + w.WriteHeader(http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusBadGateway) + }), logger) + + for _, target := range []string{"/client-error?secret=client", "/server-error?secret=server"} { + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, target, nil) + request.Header.Set("Cookie", "session=top-secret") + handler.ServeHTTP(recorder, request) + } + + inputs := logger.snapshot() + if len(inputs) != 1 { + t.Fatalf("event count = %d, want 1: %#v", len(inputs), inputs) + } + entry := inputs[0] + if entry.Level != logging.Error || entry.Source != "http" || entry.Status != http.StatusBadGateway || entry.Method != http.MethodPost || entry.Path != "/server-error" { + t.Fatalf("event = %#v", entry) + } + assertEventContainsNoSensitiveData(t, entry) +} + +func TestHTTPEventLoggingSinkFailureDoesNotChangeResponse(t *testing.T) { + handler := WithHTTPEventLogging(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte("upstream unavailable")) + }), panickingEventLogger{}) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/failure", nil)) + + if response.Code != http.StatusBadGateway || response.Body.String() != "upstream unavailable" { + t.Fatalf("response = %d %q", response.Code, response.Body.String()) + } +} + +func TestHTTPEventLoggingRecoversPanicBeforeCommitWithGenericResponseAndEvent(t *testing.T) { + logger := &recordingEventLogger{} + handler := WithHTTPEventLogging(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + panic("password=hunter2") + }), logger) + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPut, "/panic?token=query-secret", strings.NewReader("body-secret")) + request.Header.Set("Cookie", "session=cookie-secret") + + handler.ServeHTTP(recorder, request) + + if recorder.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", recorder.Code) + } + if recorder.Body.String() != "Internal Server Error\n" { + t.Fatalf("body = %q", recorder.Body.String()) + } + inputs := logger.snapshot() + if len(inputs) != 1 || inputs[0].Status != http.StatusInternalServerError || inputs[0].Message != "HTTP handler panic" { + t.Fatalf("events = %#v", inputs) + } + assertEventContainsNoSensitiveData(t, inputs[0]) +} + +func TestHTTPEventLoggingStreamsResponseBeforeHandlerCompletes(t *testing.T) { + logger := &recordingEventLogger{} + release := make(chan struct{}) + handler := WithHTTPEventLogging(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("first-chunk")) + <-release + _, _ = w.Write([]byte("second-chunk")) + }), logger) + recorder := &signalingResponseRecorder{ + ResponseRecorder: httptest.NewRecorder(), + writeObserved: make(chan struct{}), + } + done := make(chan struct{}) + go func() { + defer close(done) + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/stream", nil)) + }() + defer func() { + close(release) + <-done + }() + + select { + case <-recorder.writeObserved: + if got := recorder.Body.String(); got != "first-chunk" { + t.Fatalf("body before handler completion = %q", got) + } + case <-time.After(500 * time.Millisecond): + t.Fatal("first response chunk was buffered until handler completion") + } +} + +func TestHTTPEventLoggingCannotRewriteAlreadyCommittedResponseAfterPanic(t *testing.T) { + logger := &recordingEventLogger{} + handler := WithHTTPEventLogging(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("already-committed")) + panic("password=hunter2") + }), logger) + recorder := httptest.NewRecorder() + + func() { + defer func() { + if recovered := recover(); recovered != http.ErrAbortHandler { + t.Fatalf("panic = %#v, want http.ErrAbortHandler", recovered) + } + }() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/stream-panic", nil)) + }() + + if recorder.Code != http.StatusOK || recorder.Body.String() != "already-committed" { + t.Fatalf("committed response = %d %q", recorder.Code, recorder.Body.String()) + } + inputs := logger.snapshot() + if len(inputs) != 1 || inputs[0].Status != http.StatusInternalServerError || inputs[0].Message != "HTTP handler panic" { + t.Fatalf("events = %#v", inputs) + } +} + +type signalingResponseRecorder struct { + *httptest.ResponseRecorder + writeObserved chan struct{} + once sync.Once +} + +func (recorder *signalingResponseRecorder) Write(body []byte) (int, error) { + written, err := recorder.ResponseRecorder.Write(body) + recorder.once.Do(func() { close(recorder.writeObserved) }) + return written, err +} + +type stubTickRunner struct { + result jobs.TickResult + err error +} + +func (runner stubTickRunner) Tick(context.Context, string) (jobs.TickResult, error) { + return runner.result, runner.err +} + +func TestEventLoggingTickRunnerRecordsTickAndItemErrors(t *testing.T) { + t.Run("tick error", func(t *testing.T) { + logger := &recordingEventLogger{} + runner := WithTickEventLogging(stubTickRunner{err: errors.New("secret=provider-key")}, logger) + + _, err := runner.Tick(context.Background(), "worker-secret") + if err == nil { + t.Fatal("Tick error = nil") + } + inputs := logger.snapshot() + if len(inputs) != 1 || inputs[0].Source != "worker" || inputs[0].Level != logging.Error || inputs[0].Message != "Worker tick failed" { + t.Fatalf("events = %#v", inputs) + } + assertEventContainsNoSensitiveData(t, inputs[0]) + }) + + t.Run("item errors", func(t *testing.T) { + logger := &recordingEventLogger{} + result := jobs.TickResult{Jobs: []jobs.TickJob{ + {ID: "job-safe", Error: "password=hunter2"}, + {ID: "job-ok"}, + {ID: "job-safe-2", Error: "token=provider-key"}, + }} + runner := WithTickEventLogging(stubTickRunner{result: result}, logger) + + got, err := runner.Tick(context.Background(), "embedded-worker") + if err != nil || len(got.Jobs) != 3 { + t.Fatalf("Tick() = %#v, %v", got, err) + } + inputs := logger.snapshot() + if len(inputs) != 2 { + t.Fatalf("event count = %d, want 2: %#v", len(inputs), inputs) + } + for _, input := range inputs { + if input.Source != "worker" || input.Level != logging.Error || input.Message != "Worker job failed" { + t.Fatalf("event = %#v", input) + } + assertEventContainsNoSensitiveData(t, input) + } + }) +} + +func assertEventContainsNoSensitiveData(t *testing.T, input logging.Input) { + t.Helper() + text := strings.ToLower(input.Message + input.Method + input.Path + input.Stack) + for _, sensitive := range []string{"hunter2", "provider-key", "query-secret", "cookie-secret", "body-secret", "top-secret"} { + if strings.Contains(text, sensitive) { + t.Fatalf("event contains sensitive data %q: %#v", sensitive, input) + } + } + if input.Error != nil || input.Details != nil { + t.Fatalf("event carries unsafe error/details: %#v", input) + } +} diff --git a/backend/internal/application/runtime.go b/backend/internal/application/runtime.go new file mode 100644 index 0000000..db4c8d7 --- /dev/null +++ b/backend/internal/application/runtime.go @@ -0,0 +1,433 @@ +package application + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "net/http" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/httpapi" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/logging" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/postgres" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/providers" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/settings" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/webhook" +) + +func unavailableHandler(status int) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) }) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + +func positiveInt64Env(getenv postgres.Getenv, name string, fallback int64) int64 { + value, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64) + if err != nil || value <= 0 { + return fallback + } + return value +} + +func durationEnv(getenv postgres.Getenv, name string, fallback time.Duration) time.Duration { + milliseconds := positiveInt64Env(getenv, name, fallback.Milliseconds()) + return time.Duration(milliseconds) * time.Millisecond +} + +func parseBool(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + +func applicationJobID() string { + return applicationID("job") +} + +func applicationID(prefix string) string { + raw := make([]byte, 12) + if _, err := rand.Read(raw); err == nil { + return prefix + "-" + hex.EncodeToString(raw) + } + return prefix + "-" + strconv.FormatInt(time.Now().UnixNano(), 36) +} + +func imageEngine(getenv postgres.Getenv) string { + engine := strings.ToLower(firstNonEmpty(getenv("IMAGE_GENERATE_ENGINE"), getenv("IMAGE_CREATION_ENGINE"), getenv("IMAGE_PROVIDER"), "jimeng")) + if engine != "evolink" && engine != "bailian" { + return "jimeng" + } + return engine +} + +func videoEngine(getenv postgres.Getenv) string { + if strings.EqualFold(strings.TrimSpace(getenv("VIDEO_GENERATE_ENGINE")), "seedance") { + return "seedance" + } + return "bailian" +} + +func mockEnabled(getenv postgres.Getenv, flag string, configured bool) bool { + switch strings.ToLower(strings.TrimSpace(getenv(flag))) { + case "1", "true": + return true + case "0", "false": + return false + default: + return !configured + } +} + +func imageProvider(getenv postgres.Getenv) string { + switch imageEngine(getenv) { + case "evolink": + if mockEnabled(getenv, "EVOLINK_MOCK", strings.TrimSpace(getenv("EVOLINK_API_KEY")) != "") { + return "mock" + } + return "evolink" + case "bailian": + if mockEnabled(getenv, "BAILIAN_MOCK", bailianAPIKey(getenv) != "") { + return "mock" + } + return "bailian" + default: + configured := strings.TrimSpace(getenv("VOLCENGINE_ACCESS_KEY_ID")) != "" && strings.TrimSpace(getenv("VOLCENGINE_SECRET_ACCESS_KEY")) != "" + if mockEnabled(getenv, "JIMENG_VISUAL_MOCK", configured) { + return "mock" + } + return "volcengine-visual" + } +} + +func videoProvider(getenv postgres.Getenv) string { + if videoEngine(getenv) == "seedance" { + if mockEnabled(getenv, "SEEDANCE_MOCK", strings.TrimSpace(getenv("SEEDANCE_API_KEY")) != "") { + return "mock" + } + return "seedance" + } + if mockEnabled(getenv, "BAILIAN_MOCK", bailianAPIKey(getenv) != "") { + return "mock" + } + return "bailian" +} + +func imageModel(getenv postgres.Getenv) string { + switch imageEngine(getenv) { + case "evolink": + return firstNonEmpty(getenv("EVOLINK_IMAGE_MODEL"), "gpt-image-2") + case "bailian": + return firstNonEmpty(getenv("BAILIAN_IMAGE_MODEL"), "wan2.7-image-pro") + default: + return firstNonEmpty(getenv("JIMENG_IMAGE_GENERATE_46_REQ_KEY"), "jimeng_seedream46_cvtob") + } +} + +func videoModel(getenv postgres.Getenv) string { + if videoEngine(getenv) == "seedance" { + return firstNonEmpty(getenv("SEEDANCE_MODEL"), "doubao-seedance-2-0-260128") + } + return firstNonEmpty(getenv("BAILIAN_VIDEO_MODEL"), "wan2.7-i2v-2026-04-25") +} + +func bailianAPIKey(getenv postgres.Getenv) string { + return firstNonEmpty(getenv("BAILIAN_API_KEY"), getenv("DASHSCOPE_API_KEY")) +} + +func bailianNativeBaseURL(getenv postgres.Getenv) string { + base := strings.TrimRight(firstNonEmpty(getenv("BAILIAN_BASE_URL"), "https://llm-126wneubbdo6dbr5.cn-beijing.maas.aliyuncs.com/compatible-mode/v1"), "/") + lower := strings.ToLower(base) + const suffix = "/compatible-mode/v1" + if strings.HasSuffix(lower, suffix) { + return base[:len(base)-len(suffix)] + } + return base +} + +func buildProviderRegistry(getenv postgres.Getenv) jobs.ProviderRegistry { + client := &http.Client{Timeout: durationEnv(getenv, "ZHINIAN_PROVIDER_TIMEOUT_MS", 30*time.Second)} + maximum := positiveInt64Env(getenv, "ZHINIAN_PROVIDER_MAX_RESPONSE_BYTES", 2<<20) + return jobs.ProviderRegistry{ + "mock": providers.NewMock(firstNonEmpty(getenv("ZHINIAN_MOCK_SEED"), "zhinian")), + "volcengine-visual": providers.NewVolcengine(providers.Config{ + BaseURL: firstNonEmpty(getenv("VOLCENGINE_VISUAL_ENDPOINT"), "https://visual.volcengineapi.com"), + Model: imageModel(getenv), AccessKeyID: getenv("VOLCENGINE_ACCESS_KEY_ID"), SecretAccessKey: getenv("VOLCENGINE_SECRET_ACCESS_KEY"), + Region: firstNonEmpty(getenv("VOLCENGINE_REGION"), "cn-north-1"), Service: firstNonEmpty(getenv("VOLCENGINE_SERVICE"), "cv"), MaxResponseBytes: maximum, + }, client, nil), + "evolink": providers.NewEvoLink(providers.Config{BaseURL: firstNonEmpty(getenv("EVOLINK_BASE_URL"), "https://api.evolink.ai"), APIKey: getenv("EVOLINK_API_KEY"), Model: firstNonEmpty(getenv("EVOLINK_IMAGE_MODEL"), "gpt-image-2"), MaxResponseBytes: maximum}, client), + "bailian": providers.NewBailian(providers.Config{BaseURL: bailianNativeBaseURL(getenv), APIKey: bailianAPIKey(getenv), Model: firstNonEmpty(getenv("BAILIAN_IMAGE_MODEL"), "wan2.7-image-pro"), MaxResponseBytes: maximum}, client), + "seedance": providers.NewSeedance(providers.Config{BaseURL: firstNonEmpty(getenv("SEEDANCE_BASE_URL"), "https://ark.cn-beijing.volces.com/api/v3"), APIKey: getenv("SEEDANCE_API_KEY"), Model: firstNonEmpty(getenv("SEEDANCE_MODEL"), "doubao-seedance-2-0-260128"), MaxResponseBytes: maximum}, client), + } +} + +func providerImageTargets(getenv postgres.Getenv) map[string]jobs.ProviderTarget { + jimengConfigured := strings.TrimSpace(getenv("VOLCENGINE_ACCESS_KEY_ID")) != "" && strings.TrimSpace(getenv("VOLCENGINE_SECRET_ACCESS_KEY")) != "" + return map[string]jobs.ProviderTarget{ + "jimeng": { + Provider: providerOrMock("volcengine-visual", mockEnabled(getenv, "JIMENG_VISUAL_MOCK", jimengConfigured)), + Model: firstNonEmpty(getenv("JIMENG_IMAGE_GENERATE_46_REQ_KEY"), "jimeng_seedream46_cvtob"), + }, + "evolink": { + Provider: providerOrMock("evolink", mockEnabled(getenv, "EVOLINK_MOCK", strings.TrimSpace(getenv("EVOLINK_API_KEY")) != "")), + Model: firstNonEmpty(getenv("EVOLINK_IMAGE_MODEL"), "gpt-image-2"), + }, + "bailian": { + Provider: providerOrMock("bailian", mockEnabled(getenv, "BAILIAN_MOCK", bailianAPIKey(getenv) != "")), + Model: firstNonEmpty(getenv("BAILIAN_IMAGE_MODEL"), "wan2.7-image-pro"), + }, + } +} + +func providerVideoTargets(getenv postgres.Getenv) map[string]jobs.ProviderTarget { + return map[string]jobs.ProviderTarget{ + "seedance": { + Provider: providerOrMock("seedance", mockEnabled(getenv, "SEEDANCE_MOCK", strings.TrimSpace(getenv("SEEDANCE_API_KEY")) != "")), + Model: firstNonEmpty(getenv("SEEDANCE_MODEL"), "doubao-seedance-2-0-260128"), + Settings: map[string]any{ + "ratio": firstNonEmpty(getenv("SEEDANCE_DEFAULT_RATIO"), "9:16"), + "duration": float64(positiveInt64Env(getenv, "SEEDANCE_DEFAULT_DURATION", 5)), + "resolution": firstNonEmpty(getenv("SEEDANCE_DEFAULT_RESOLUTION"), "720p"), + }, + }, + "bailian": { + Provider: providerOrMock("bailian", mockEnabled(getenv, "BAILIAN_MOCK", bailianAPIKey(getenv) != "")), + Model: firstNonEmpty(getenv("BAILIAN_VIDEO_MODEL"), "wan2.7-i2v-2026-04-25"), + }, + } +} + +func providerOrMock(provider string, mocked bool) string { + if mocked { + return "mock" + } + return provider +} + +func defaultWebhookSender(getenv postgres.Getenv) (*webhook.HTTPSender, error) { + return webhook.NewPublicHTTPSender( + durationEnv(getenv, "ZHINIAN_WEBHOOK_TIMEOUT_MS", 10*time.Second), + webhook.NewPublicDestinationPolicy(nil, nil), + ) +} + +func capabilitySummary(getenv postgres.Getenv) func(context.Context) (any, error) { + return func(context.Context) (any, error) { + engine := imageEngine(getenv) + provider := map[string]string{"jimeng": "volcengine-visual", "evolink": "evolink", "bailian": "bailian"}[engine] + primaryVideoEngine := videoEngine(getenv) + primaryVideo := map[string]any{ + "id": "video.generate", "kind": "video", "engine": primaryVideoEngine, + "provider": primaryVideoEngine, "reqKey": videoModel(getenv), + } + if primaryVideoEngine == "seedance" { + primaryVideo["label"] = "Seedance 视频生成" + primaryVideo["limits"] = seedanceCapabilityLimits() + } else { + primaryVideo["label"] = "百炼图生视频" + primaryVideo["limits"] = bailianVideoCapabilityLimits() + } + return []any{ + map[string]any{"id": "image.generate", "label": "图片生成 4.6", "kind": "image", "engine": engine, "provider": provider, "reqKey": imageModel(getenv)}, + primaryVideo, + map[string]any{"id": "video.generate.bailian", "label": "百炼图生视频", "kind": "video", "engine": "bailian", "provider": "bailian", "reqKey": firstNonEmpty(getenv("BAILIAN_VIDEO_MODEL"), "wan2.7-i2v-2026-04-25"), "limits": bailianVideoCapabilityLimits()}, + }, nil + } +} + +func seedanceCapabilityLimits() map[string]any { + return map[string]any{"durationSeconds": map[string]int{"min": 4, "max": 15}, "ratios": []string{"16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"}, "resolutions": []string{"480p", "720p", "1080p"}} +} + +func bailianVideoCapabilityLimits() map[string]any { + return map[string]any{"inputImages": map[string]int{"min": 1, "max": 2}, "durationSeconds": map[string]int{"min": 2, "max": 15}, "resolutions": []string{"720P", "1080P"}} +} + +func runtimeHealthDetails(getenv postgres.Getenv) httpapi.HealthDetails { + image := imageEngine(getenv) + imageLabel := map[string]string{"jimeng": "即梦", "evolink": "EvoLink", "bailian": "阿里云百炼"}[image] + visualConfigured := strings.TrimSpace(getenv("VOLCENGINE_ACCESS_KEY_ID")) != "" && strings.TrimSpace(getenv("VOLCENGINE_SECRET_ACCESS_KEY")) != "" + auth, _ := ParseAuthConfig(getenv) + bailianKey := bailianAPIKey(getenv) + bailianMode := "missing" + if mockFlagEnabled(getenv("BAILIAN_MOCK")) { + bailianMode = "mock" + } else if bailianKey != "" { + bailianMode = "bailian" + } + authMode := "disabled" + if auth.Required { + authMode = "missing" + if auth.Configured { + authMode = "configured" + } + } + return httpapi.HealthDetails{ + VisualAPIMode: providerMode(mockEnabled(getenv, "JIMENG_VISUAL_MOCK", visualConfigured), "volcengine"), + EvolinkMode: providerMode(mockEnabled(getenv, "EVOLINK_MOCK", strings.TrimSpace(getenv("EVOLINK_API_KEY")) != ""), "evolink"), + SeedanceMode: providerMode(mockEnabled(getenv, "SEEDANCE_MOCK", strings.TrimSpace(getenv("SEEDANCE_API_KEY")) != ""), "seedance"), + BailianMode: bailianMode, + AuthMode: authMode, + Capabilities: []any{ + map[string]any{"id": "image.generate", "label": "图片生成 4.6", "engine": image, "engineLabel": imageLabel, "reqKey": imageModel(getenv)}, + map[string]any{"id": "video.generate", "label": "Seedance 视频生成", "engine": "seedance", "engineLabel": "Seedance", "reqKey": firstNonEmpty(getenv("SEEDANCE_MODEL"), "doubao-seedance-2-0-260128")}, + }, + } +} + +func providerMode(mock bool, live string) string { + if mock { + return "mock" + } + return live +} + +func mockFlagEnabled(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "1", "true": + return true + default: + return false + } +} + +func remoteAssetMaxBytes(getenv postgres.Getenv) int64 { + return positiveInt64Env(getenv, "ZHINIAN_REMOTE_ASSET_MAX_BYTES", 20<<20) +} + +var runtimeSettingKeys = []string{ + "ALI_OSS_ACCESS_KEY_ID", "ALI_OSS_ACCESS_KEY_SECRET", "ALI_OSS_BUCKET", "ALI_OSS_ENDPOINT", "ALI_OSS_PUBLIC_BASE_URL", + "BAILIAN_API_KEY", "BAILIAN_BASE_URL", "BAILIAN_IMAGE_MODEL", "BAILIAN_VIDEO_MODEL", "DATABASE_URL", "DASHSCOPE_API_KEY", + "EVOLINK_API_KEY", "EVOLINK_BASE_URL", "EVOLINK_IMAGE_MODEL", "EVOLINK_IMAGE_QUALITY", "IMAGE_GENERATE_ENGINE", + "SEEDANCE_API_KEY", "SEEDANCE_MODEL", "VIDEO_GENERATE_ENGINE", "VOLCENGINE_ACCESS_KEY_ID", "VOLCENGINE_SECRET_ACCESS_KEY", + "ZHINIAN_AUTH_REQUIRED", "ZHINIAN_AUTH_SESSION_SECRET", "ZHINIAN_BILLING_ACCOUNT_BANK", "ZHINIAN_BILLING_ACCOUNT_NAME", + "ZHINIAN_BILLING_ACCOUNT_NUMBER", "ZHINIAN_BILLING_CONTACT", "ZHINIAN_BILLING_REQUIRED", +} + +func runtimeEnvironment(getenv postgres.Getenv) map[string]string { + values := make(map[string]string, len(runtimeSettingKeys)) + for _, key := range runtimeSettingKeys { + if value := getenv(key); value != "" { + values[key] = value + } + } + return values +} + +func defaultSettingsService(getenv postgres.Getenv) *settings.Service { + path := firstNonEmpty(getenv("ZHINIAN_SETTINGS_FILE"), ".env.local") + return settings.New(path, runtimeEnvironment(getenv), nil) +} + +type settingsBillingAccountStore struct{ service *settings.Service } + +func (store settingsBillingAccountStore) Load(ctx context.Context) (billing.AccountConfig, error) { + value, err := store.service.Get(ctx) + if err != nil { + return billing.AccountConfig{}, err + } + payload, ok := value.(settings.Payload) + if !ok { + return billing.AccountConfig{}, fmt.Errorf("load billing account settings: unexpected settings payload") + } + fields := map[string]string{} + for _, group := range payload.Groups { + for _, field := range group.Fields { + fields[field.Key] = field.Value + } + } + return billing.AccountConfig{ + AccountName: fields["ZHINIAN_BILLING_ACCOUNT_NAME"], BankName: fields["ZHINIAN_BILLING_ACCOUNT_BANK"], + AccountNumber: fields["ZHINIAN_BILLING_ACCOUNT_NUMBER"], Contact: fields["ZHINIAN_BILLING_CONTACT"], + }, nil +} + +func (store settingsBillingAccountStore) Save(ctx context.Context, value billing.AccountConfig) error { + _, err := store.service.Save(ctx, map[string]any{ + "ZHINIAN_BILLING_ACCOUNT_NAME": value.AccountName, "ZHINIAN_BILLING_ACCOUNT_BANK": value.BankName, + "ZHINIAN_BILLING_ACCOUNT_NUMBER": value.AccountNumber, "ZHINIAN_BILLING_CONTACT": value.Contact, + }) + return err +} + +type logServiceAdapter struct{ service *logging.Service } + +func (adapter logServiceAdapter) List(ctx context.Context, filters httpapi.LogFilters) (any, error) { + return adapter.service.List(ctx, logging.Filters{Level: filters.Level, Q: filters.Q, Source: filters.Source, Limit: filters.Limit}) +} + +func (adapter logServiceAdapter) Clear(ctx context.Context) error { return adapter.service.Clear(ctx) } + +func (adapter logServiceAdapter) Append(ctx context.Context, input logging.Input) (logging.Entry, error) { + return adapter.service.Append(ctx, input) +} + +func defaultLogService(getenv postgres.Getenv) httpapi.LogService { + runtimeDirectory := firstNonEmpty(getenv("ZHINIAN_RUNTIME_DIR"), ".runtime") + logDirectory := firstNonEmpty(getenv("ZHINIAN_LOG_DIR"), filepath.Join(runtimeDirectory, "logs")) + return logServiceAdapter{service: logging.New(filepath.Join(logDirectory, "server-events.jsonl"), positiveInt64Env(getenv, "ZHINIAN_LOG_MAX_BYTES", 5<<20), nil, nil)} +} + +type prefixedBlobStore struct { + prefix string + store assets.BlobStore +} + +func (store prefixedBlobStore) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (assets.StoredObject, error) { + stored, err := store.store.Put(ctx, path.Join(store.prefix, key), body, size, contentType) + if err != nil { + return assets.StoredObject{}, err + } + // StoragePath is an application key used by /uploads and + // /generated-results. The OSS namespace prefix is private to this adapter. + stored.Key = key + return stored, nil +} + +func (store prefixedBlobStore) Read(ctx context.Context, key string) (assets.Blob, error) { + return store.store.Read(ctx, path.Join(store.prefix, key)) +} + +func (store prefixedBlobStore) Delete(ctx context.Context, key string) error { + return store.store.Delete(ctx, path.Join(store.prefix, key)) +} + +func configuredOSSBlobStore(getenv postgres.Getenv) (assets.BlobStore, bool, error) { + endpoint, bucket := strings.TrimSpace(getenv("ALI_OSS_ENDPOINT")), strings.TrimSpace(getenv("ALI_OSS_BUCKET")) + accessKeyID, secret := strings.TrimSpace(getenv("ALI_OSS_ACCESS_KEY_ID")), strings.TrimSpace(getenv("ALI_OSS_ACCESS_KEY_SECRET")) + publicURL := strings.TrimSpace(getenv("ALI_OSS_PUBLIC_BASE_URL")) + if endpoint == "" || bucket == "" || accessKeyID == "" || secret == "" || publicURL == "" { + return nil, false, nil + } + client, err := assets.NewOSSHTTPClient(accessKeyID, secret, &http.Client{Timeout: durationEnv(getenv, "ZHINIAN_OSS_TIMEOUT_MS", 30*time.Second)}, nil) + if err != nil { + return nil, false, err + } + store, err := assets.NewOSS(assets.OSSConfig{Endpoint: endpoint, Bucket: bucket, PublicBaseURL: publicURL, PublicRead: true}, client) + if err != nil { + return nil, false, err + } + prefix := strings.Trim(strings.TrimSpace(getenv("ALI_OSS_PREFIX")), "/") + if prefix == "" { + prefix = "zhinian" + } + return prefixedBlobStore{prefix: prefix, store: store}, true, nil +} diff --git a/backend/internal/application/runtime_test.go b/backend/internal/application/runtime_test.go new file mode 100644 index 0000000..e1e723a --- /dev/null +++ b/backend/internal/application/runtime_test.go @@ -0,0 +1,179 @@ +package application + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "reflect" + "testing" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/settings" +) + +func TestDefaultSettingsServicePersistsRestartRequiredWithoutPartialHotReload(t *testing.T) { + path := filepath.Join(t.TempDir(), ".env.local") + key := "IMAGE_GENERATE_ENGINE" + service := defaultSettingsService(func(name string) string { + if name == "ZHINIAN_SETTINGS_FILE" { + return path + } + return "" + }) + value, err := service.Save(context.Background(), map[string]any{key: "evolink"}) + if err != nil { + t.Fatal(err) + } + payload, ok := value.(settings.Payload) + if !ok || !payload.RestartRequired { + t.Fatalf("payload=%#v", value) + } +} + +func TestBillingAccountAndSettingsUseOneRuntimeSource(t *testing.T) { + path := filepath.Join(t.TempDir(), ".env.local") + if err := os.WriteFile(path, []byte("ZHINIAN_BILLING_ACCOUNT_NAME=Original\nZHINIAN_BILLING_ACCOUNT_BANK=Old Bank\n"), 0o600); err != nil { + t.Fatal(err) + } + service := defaultSettingsService(func(name string) string { + if name == "ZHINIAN_SETTINGS_FILE" { + return path + } + return "" + }) + store := settingsBillingAccountStore{service: service} + if err := store.Save(context.Background(), billing.AccountConfig{AccountName: "Updated", BankName: "New Bank", AccountNumber: "123", Contact: "Ops"}); err != nil { + t.Fatal(err) + } + loaded, err := store.Load(context.Background()) + if err != nil || loaded.AccountName != "Updated" || loaded.BankName != "New Bank" || loaded.AccountNumber != "123" || loaded.Contact != "Ops" { + t.Fatalf("loaded=%#v err=%v", loaded, err) + } + payload, err := service.Get(context.Background()) + if err != nil || fieldValueFromSettings(t, payload, "ZHINIAN_BILLING_ACCOUNT_NAME") != "Updated" { + t.Fatalf("settings payload=%#v err=%v", payload, err) + } +} + +func fieldValueFromSettings(t *testing.T, value any, key string) string { + t.Helper() + payload := value.(settings.Payload) + for _, group := range payload.Groups { + for _, field := range group.Fields { + if field.Key == key { + return field.Value + } + } + } + t.Fatalf("settings field %s not found", key) + return "" +} + +func TestRuntimeHealthDetailsMatchTypeScriptDefaultsAndConfiguredModes(t *testing.T) { + values := map[string]string{ + "VOLCENGINE_ACCESS_KEY_ID": "access", + "VOLCENGINE_SECRET_ACCESS_KEY": "secret", + "SEEDANCE_API_KEY": "seedance-key", + "ZHINIAN_AUTH_REQUIRED": "true", + "ZHINIAN_AUTH_SESSION_SECRET": "session-secret", + } + details := runtimeHealthDetails(func(name string) string { return values[name] }) + if details.VisualAPIMode != "volcengine" || details.EvolinkMode != "mock" || details.SeedanceMode != "seedance" || details.BailianMode != "missing" || details.AuthMode != "configured" { + t.Fatalf("details = %+v", details) + } + if len(details.Capabilities) != 2 { + t.Fatalf("capabilities = %#v, want image and Seedance", details.Capabilities) + } + image := details.Capabilities[0].(map[string]any) + video := details.Capabilities[1].(map[string]any) + if image["id"] != "image.generate" || image["engineLabel"] != "即梦" || video["id"] != "video.generate" || video["engineLabel"] != "Seedance" { + t.Fatalf("capabilities = %#v", details.Capabilities) + } +} + +func TestRuntimeHealthDetailsHonorExplicitMockFlagsAndImageEngine(t *testing.T) { + values := map[string]string{ + "IMAGE_GENERATE_ENGINE": "bailian", + "BAILIAN_API_KEY": "bailian-key", + "BAILIAN_MOCK": "true", + "JIMENG_VISUAL_MOCK": "true", + "EVOLINK_MOCK": "false", + "SEEDANCE_MOCK": "true", + } + details := runtimeHealthDetails(func(name string) string { return values[name] }) + if details.VisualAPIMode != "mock" || details.EvolinkMode != "evolink" || details.SeedanceMode != "mock" || details.BailianMode != "mock" || details.AuthMode != "disabled" { + t.Fatalf("details = %+v", details) + } + image := details.Capabilities[0].(map[string]any) + if image["engine"] != "bailian" || image["engineLabel"] != "阿里云百炼" || image["reqKey"] != "wan2.7-image-pro" { + t.Fatalf("image capability = %#v", image) + } +} + +func TestCapabilitySummaryMatchesConfiguredDefaultVideoEngine(t *testing.T) { + for _, test := range []struct { + name, engine, wantEngine, wantProvider, wantModel string + }{ + {name: "Bailian default", engine: "", wantEngine: "bailian", wantProvider: "bailian", wantModel: "wan2.7-i2v-2026-04-25"}, + {name: "Seedance configured", engine: "seedance", wantEngine: "seedance", wantProvider: "seedance", wantModel: "doubao-seedance-2-0-260128"}, + } { + t.Run(test.name, func(t *testing.T) { + values := map[string]string{"VIDEO_GENERATE_ENGINE": test.engine} + value, err := capabilitySummary(func(name string) string { return values[name] })(context.Background()) + if err != nil { + t.Fatal(err) + } + capabilities := value.([]any) + video := capabilities[1].(map[string]any) + if video["engine"] != test.wantEngine || video["provider"] != test.wantProvider || video["reqKey"] != test.wantModel { + t.Fatalf("video capability = %#v", video) + } + }) + } +} + +func TestRemoteAssetMaximumDefaultsToTwentyMiB(t *testing.T) { + getenv := func(string) string { return "" } + if got := remoteAssetMaxBytes(getenv); got != 20<<20 { + t.Fatalf("remote asset maximum = %d, want %d", got, int64(20<<20)) + } + getenv = func(string) string { return "3145728" } + if got := remoteAssetMaxBytes(getenv); got != 3<<20 { + t.Fatalf("configured remote asset maximum = %d, want %d", got, int64(3<<20)) + } +} + +func TestPrefixedBlobStoreKeepsApplicationStoragePathStable(t *testing.T) { + inner := &recordingBlobStore{} + store := prefixedBlobStore{prefix: "tenant-prefix", store: inner} + stored, err := store.Put(context.Background(), "uploads/day/file.png", bytes.NewReader([]byte("x")), 1, "image/png") + if err != nil { + t.Fatal(err) + } + if stored.Key != "uploads/day/file.png" || inner.putKey != "tenant-prefix/uploads/day/file.png" { + t.Fatalf("stored=%#v inner=%q", stored, inner.putKey) + } + _, _ = store.Read(context.Background(), stored.Key) + _ = store.Delete(context.Background(), stored.Key) + if !reflect.DeepEqual([]string{inner.readKey, inner.deleteKey}, []string{"tenant-prefix/uploads/day/file.png", "tenant-prefix/uploads/day/file.png"}) { + t.Fatalf("read/delete=%q/%q", inner.readKey, inner.deleteKey) + } +} + +type recordingBlobStore struct{ putKey, readKey, deleteKey string } + +func (s *recordingBlobStore) Put(_ context.Context, key string, _ io.Reader, _ int64, _ string) (assets.StoredObject, error) { + s.putKey = key + return assets.StoredObject{Key: key, URL: "https://cdn.example/" + key}, nil +} +func (s *recordingBlobStore) Read(_ context.Context, key string) (assets.Blob, error) { + s.readKey = key + return assets.Blob{Body: io.NopCloser(bytes.NewReader(nil))}, nil +} +func (s *recordingBlobStore) Delete(_ context.Context, key string) error { + s.deleteKey = key + return nil +} diff --git a/backend/internal/assets/assets.go b/backend/internal/assets/assets.go index cd0de49..5f5fde1 100644 --- a/backend/internal/assets/assets.go +++ b/backend/internal/assets/assets.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "html" "io" "path" "regexp" @@ -111,6 +112,10 @@ type Catalog interface { DeleteOwner(context.Context, string, string) (Asset, bool, error) } +type storagePathCatalog interface { + GetOwnerByStoragePath(context.Context, string, string) (Asset, bool, error) +} + type StoredObject struct{ Key, URL string } type Blob struct { Body io.ReadCloser @@ -177,6 +182,7 @@ func (s *Service) Get(ctx context.Context, scope Scope, id string) (Asset, error type CreateExternalCommand struct { URL, Name string Kind Kind + Source Source Tags []string } @@ -192,6 +198,10 @@ func (s *Service) CreateExternal(ctx context.Context, scope Scope, cmd CreateExt if kind == "" { kind = KindImage } + source := cmd.Source + if source == "" { + source = SourceExternal + } name := cmd.Name tags := cloneStrings(cmd.Tags) metadata := map[string]any{} @@ -207,7 +217,7 @@ func (s *Service) CreateExternal(ctx context.Context, scope Scope, cmd CreateExt } metadata["registeredFrom"] = "api" } - a := Asset{ID: s.id("asset"), OwnerID: scope.ownerID, Kind: kind, Name: name, URL: cmd.URL, Source: SourceExternal, Tags: tags, Metadata: metadata, CreatedAt: now, UpdatedAt: now} + a := Asset{ID: s.id("asset"), OwnerID: scope.ownerID, Kind: kind, Name: name, URL: cmd.URL, Source: source, Tags: tags, Metadata: metadata, CreatedAt: now, UpdatedAt: now} return s.catalog.Create(ctx, a) } @@ -218,6 +228,24 @@ type UploadCommand struct { Tags []string } +type ImportGeneratedCommand struct { + URL, Name, Capability, JobID string + Kind Kind + Source Source + Tags []string + Metadata map[string]any +} + +// ImportMockCommand describes a locally generated development result. Unlike +// ImportGeneratedCommand it carries no URL, so mock output never enters the +// remote-fetch/SSRF boundary. +type ImportMockCommand struct { + Name, Capability, JobID string + Kind Kind + Tags []string + Metadata map[string]any +} + func (s *Service) Upload(ctx context.Context, scope Scope, cmd UploadCommand) (Asset, error) { if err := validScope(scope); err != nil { return Asset{}, err @@ -251,6 +279,110 @@ func (s *Service) Upload(ctx context.Context, scope Scope, cmd UploadCommand) (A return created, nil } +// ImportGenerated fetches a provider result through the bounded RemoteFetcher, +// stores it in the configured BlobStore, and only then creates relational +// metadata. Upload already owns blob compensation when catalog persistence +// fails, so provider outputs share the same crash-safe write ordering. +func (s *Service) ImportGenerated(ctx context.Context, scope Scope, cmd ImportGeneratedCommand) (Asset, error) { + if strings.TrimSpace(cmd.URL) == "" || s.remote == nil { + return Asset{}, ErrBlobNotFound + } + blob, err := s.remote.Fetch(ctx, cmd.URL) + if err != nil { + return Asset{}, err + } + defer blob.Body.Close() + maximum := blob.Size + if maximum < 0 { + maximum = 64 << 20 + } + content, err := io.ReadAll(io.LimitReader(blob.Body, maximum+1)) + if err != nil || int64(len(content)) > maximum { + return Asset{}, ErrRemoteTooLarge + } + name := cmd.Name + if strings.TrimSpace(name) == "" { + name = path.Base(strings.SplitN(cmd.URL, "?", 2)[0]) + } + if s.blobs == nil { + return Asset{}, errors.New("blob store is unavailable") + } + cleanName := sanitizeFileName(name) + key := path.Join("generated-results", s.now().UTC().Format("2006-01-02"), s.id("file")+"-"+cleanName) + stored, err := s.blobs.Put(ctx, key, bytes.NewReader(content), int64(len(content)), blob.ContentType) + if err != nil { + return Asset{}, err + } + kind := cmd.Kind + if kind == "" { + kind = inferKind(blob.ContentType) + } + now := s.now().UTC() + source := cmd.Source + if source == "" { + source = SourceGenerated + } + metadata := cloneMetadata(cmd.Metadata) + metadata["contentType"] = blob.ContentType + metadata["size"] = len(content) + metadata["capability"] = cmd.Capability + metadata["jobId"] = cmd.JobID + metadata["importedFrom"] = cmd.URL + asset := Asset{ID: s.id("asset"), OwnerID: scope.ownerID, Kind: kind, Name: name, URL: stored.URL, StoragePath: stored.Key, Source: source, Tags: cloneStrings(cmd.Tags), Metadata: metadata, CreatedAt: now, UpdatedAt: now} + created, err := s.catalog.Create(ctx, asset) + if err != nil { + _ = s.blobs.Delete(context.WithoutCancel(ctx), stored.Key) + return Asset{}, err + } + return created, nil +} + +// ImportMock stores a small deterministic placeholder in the configured blob +// store and registers it as a generated asset. It is intentionally a separate +// path from remote imports: relative mock provider URLs are never interpreted +// as fetch destinations. +func (s *Service) ImportMock(ctx context.Context, scope Scope, cmd ImportMockCommand) (Asset, error) { + if err := validScope(scope); err != nil { + return Asset{}, err + } + if s.blobs == nil { + return Asset{}, errors.New("blob store is unavailable") + } + kind := cmd.Kind + if kind == "" { + kind = KindImage + } + name, contentType, content := mockOutput(cmd.Name, kind, cmd.JobID) + key := path.Join("generated-results", s.now().UTC().Format("2006-01-02"), s.id("file")+"-"+sanitizeFileName(name)) + stored, err := s.blobs.Put(ctx, key, bytes.NewReader(content), int64(len(content)), contentType) + if err != nil { + return Asset{}, err + } + now := s.now().UTC() + metadata := cloneMetadata(cmd.Metadata) + metadata["contentType"] = contentType + metadata["size"] = len(content) + metadata["capability"] = cmd.Capability + metadata["jobId"] = cmd.JobID + metadata["mock"] = true + asset := Asset{ID: s.id("asset"), OwnerID: scope.ownerID, Kind: kind, Name: name, URL: stored.URL, StoragePath: stored.Key, Source: SourceGenerated, Tags: cloneStrings(cmd.Tags), Metadata: metadata, CreatedAt: now, UpdatedAt: now} + created, err := s.catalog.Create(ctx, asset) + if err != nil { + _ = s.blobs.Delete(context.WithoutCancel(ctx), stored.Key) + return Asset{}, err + } + return created, nil +} + +func mockOutput(name string, kind Kind, jobID string) (string, string, []byte) { + if kind == KindVideo { + return defaultString(name, "mock-video.mp4"), "video/mp4", []byte("mock video result for " + jobID + "\n") + } + name = defaultString(name, "mock-image.svg") + content := `Mock image` + html.EscapeString(jobID) + `` + return name, "image/svg+xml", []byte(content) +} + func (s *Service) Delete(ctx context.Context, scope Scope, id string) (Asset, error) { a, err := s.Get(ctx, scope, id) if err != nil { @@ -290,6 +422,30 @@ func (s *Service) Download(ctx context.Context, scope Scope, id string) (Blob, e return s.remote.Fetch(ctx, a.URL) } +// DownloadPath serves a stored object only after its metadata has been found +// in the requesting owner's catalog. Catalogs that support HTTP file serving +// implement the optional storage-path lookup without widening other callers. +func (s *Service) DownloadPath(ctx context.Context, scope Scope, storagePath string) (Blob, error) { + if err := validScope(scope); err != nil { + return Blob{}, err + } + lookup, ok := s.catalog.(storagePathCatalog) + if !ok { + return Blob{}, ErrNotFound + } + a, found, err := lookup.GetOwnerByStoragePath(ctx, scope.ownerID, storagePath) + if err != nil { + return Blob{}, err + } + if !found || a.StoragePath == "" { + return Blob{}, ErrNotFound + } + if s.blobs == nil { + return Blob{}, ErrBlobNotFound + } + return s.blobs.Read(ctx, a.StoragePath) +} + func validScope(s Scope) error { if strings.TrimSpace(s.ownerID) == "" { return errors.New("asset owner is required") @@ -319,6 +475,13 @@ func cloneStrings(values []string) []string { } return append([]string{}, values...) } +func cloneMetadata(values map[string]any) map[string]any { + result := make(map[string]any, len(values)+5) + for key, value := range values { + result[key] = value + } + return result +} func contains(values []string, want string) bool { for _, v := range values { if v == want { diff --git a/backend/internal/assets/oss.go b/backend/internal/assets/oss.go new file mode 100644 index 0000000..5321d8c --- /dev/null +++ b/backend/internal/assets/oss.go @@ -0,0 +1,136 @@ +package assets + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "path" + "strings" +) + +const OSSACLPublicRead = "public-read" + +type OSSConfig struct { + Endpoint string + Bucket string + PublicBaseURL string + PublicRead bool +} + +type OSSPutRequest struct { + Endpoint, Bucket, Key, ContentType string + Body io.Reader + Size int64 +} +type OSSObjectRequest struct{ Endpoint, Bucket, Key string } +type OSSACLRequest struct{ Endpoint, Bucket, Key, ACL string } + +// OSSClient is the SDK-independent seam. Runtime composition can adapt any OSS +// SDK while tests remain deterministic and credential/network free. +type OSSClient interface { + Put(context.Context, OSSPutRequest) error + SetACL(context.Context, OSSACLRequest) error + Get(context.Context, OSSObjectRequest) (Blob, error) + Delete(context.Context, OSSObjectRequest) error +} + +type OSSError struct { + Status int + Code string + Err error +} + +func (e *OSSError) Error() string { + if e.Err != nil { + return e.Err.Error() + } + if e.Code != "" { + return e.Code + } + return "OSS request failed" +} +func (e *OSSError) Unwrap() error { return e.Err } + +type OSS struct { + config OSSConfig + client OSSClient +} + +func NewOSS(config OSSConfig, client OSSClient) (*OSS, error) { + config.Endpoint = strings.TrimRight(strings.TrimSpace(config.Endpoint), "/") + config.Bucket = strings.TrimSpace(config.Bucket) + config.PublicBaseURL = strings.TrimRight(strings.TrimSpace(config.PublicBaseURL), "/") + if config.Endpoint == "" || config.Bucket == "" || config.PublicBaseURL == "" || client == nil { + return nil, errors.New("OSS endpoint, bucket, public URL, and client are required") + } + if parsed, err := url.Parse(config.PublicBaseURL); err != nil || parsed.Scheme == "" || parsed.Host == "" { + return nil, errors.New("OSS public URL must be absolute") + } + return &OSS{config: config, client: client}, nil +} + +func (s *OSS) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (StoredObject, error) { + if err := validOSSKey(key); err != nil { + return StoredObject{}, err + } + err := s.client.Put(ctx, OSSPutRequest{Endpoint: s.config.Endpoint, Bucket: s.config.Bucket, Key: key, Body: body, Size: size, ContentType: contentType}) + if err != nil { + return StoredObject{}, mapOSSError(err) + } + if s.config.PublicRead { + if err = s.client.SetACL(ctx, OSSACLRequest{Endpoint: s.config.Endpoint, Bucket: s.config.Bucket, Key: key, ACL: OSSACLPublicRead}); err != nil { + return StoredObject{}, mapOSSError(err) + } + } + return StoredObject{Key: key, URL: s.config.PublicBaseURL + "/" + escapeOSSKey(key)}, nil +} +func (s *OSS) Read(ctx context.Context, key string) (Blob, error) { + if err := validOSSKey(key); err != nil { + return Blob{}, err + } + blob, err := s.client.Get(ctx, OSSObjectRequest{Endpoint: s.config.Endpoint, Bucket: s.config.Bucket, Key: key}) + if err != nil { + return Blob{}, mapOSSError(err) + } + return blob, nil +} +func (s *OSS) Delete(ctx context.Context, key string) error { + if err := validOSSKey(key); err != nil { + return err + } + err := s.client.Delete(ctx, OSSObjectRequest{Endpoint: s.config.Endpoint, Bucket: s.config.Bucket, Key: key}) + if isMissingOSS(err) { + return nil + } + return mapOSSError(err) +} +func mapOSSError(err error) error { + if err == nil { + return nil + } + if isMissingOSS(err) { + return ErrBlobNotFound + } + return fmt.Errorf("OSS operation failed: %w", err) +} +func isMissingOSS(err error) bool { + var ossErr *OSSError + return errors.As(err, &ossErr) && (ossErr.Status == 404 || ossErr.Code == "NoSuchKey" || ossErr.Code == "NoSuchObject") +} +func validOSSKey(key string) error { + if key == "" || strings.HasPrefix(key, "/") || strings.Contains(key, "\\") || path.Clean(key) != key || key == "." || strings.HasPrefix(key, "../") { + return ErrUnsafeBlobKey + } + return nil +} +func escapeOSSKey(key string) string { + parts := strings.Split(key, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + return strings.Join(parts, "/") +} + +var _ BlobStore = (*OSS)(nil) diff --git a/backend/internal/assets/oss_http.go b/backend/internal/assets/oss_http.go new file mode 100644 index 0000000..294d973 --- /dev/null +++ b/backend/internal/assets/oss_http.go @@ -0,0 +1,237 @@ +package assets + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "sort" + "strings" + "time" +) + +const maxOSSErrorBody = 64 << 10 + +// OSSHTTPClient implements OSSClient using Aliyun OSS's HTTP authorization +// protocol. The supplied HTTP client owns timeout and transport policy. +type OSSHTTPClient struct { + accessKeyID string + accessKeySecret string + client *http.Client + now func() time.Time +} + +// NewOSSHTTPClient constructs an OSS adapter with injected credentials, +// transport, and clock. Callers should supply an http.Client with a bounded +// Timeout. +func NewOSSHTTPClient(accessKeyID, accessKeySecret string, client *http.Client, now func() time.Time) (*OSSHTTPClient, error) { + if strings.TrimSpace(accessKeyID) == "" || strings.TrimSpace(accessKeySecret) == "" { + return nil, errors.New("OSS access key ID and secret are required") + } + if client == nil { + return nil, errors.New("OSS HTTP client is required") + } + if now == nil { + now = time.Now + } + return &OSSHTTPClient{accessKeyID: accessKeyID, accessKeySecret: accessKeySecret, client: client, now: now}, nil +} + +func (c *OSSHTTPClient) Put(ctx context.Context, request OSSPutRequest) error { + req, err := c.newRequest(ctx, http.MethodPut, request.Endpoint, request.Bucket, request.Key, "", request.Body) + if err != nil { + return err + } + if request.Size < 0 { + return errors.New("OSS object size must not be negative") + } + req.ContentLength = request.Size + req.Header.Set("Content-Length", fmt.Sprintf("%d", request.Size)) + if request.ContentType != "" { + req.Header.Set("Content-Type", request.ContentType) + } + c.sign(req, request.Bucket, request.Key, "") + response, err := c.do(req) + if err != nil { + return err + } + return consumeOSSResponse(response) +} + +func (c *OSSHTTPClient) SetACL(ctx context.Context, request OSSACLRequest) error { + req, err := c.newRequest(ctx, http.MethodPut, request.Endpoint, request.Bucket, request.Key, "acl", nil) + if err != nil { + return err + } + req.Header.Set("x-oss-object-acl", request.ACL) + c.sign(req, request.Bucket, request.Key, "acl") + response, err := c.do(req) + if err != nil { + return err + } + return consumeOSSResponse(response) +} + +func (c *OSSHTTPClient) Get(ctx context.Context, request OSSObjectRequest) (Blob, error) { + req, err := c.newRequest(ctx, http.MethodGet, request.Endpoint, request.Bucket, request.Key, "", nil) + if err != nil { + return Blob{}, err + } + c.sign(req, request.Bucket, request.Key, "") + response, err := c.do(req) + if err != nil { + return Blob{}, err + } + if !successfulOSSStatus(response.StatusCode) { + return Blob{}, readOSSError(response) + } + return Blob{Body: response.Body, ContentType: response.Header.Get("Content-Type"), Size: response.ContentLength}, nil +} + +func (c *OSSHTTPClient) Delete(ctx context.Context, request OSSObjectRequest) error { + req, err := c.newRequest(ctx, http.MethodDelete, request.Endpoint, request.Bucket, request.Key, "", nil) + if err != nil { + return err + } + c.sign(req, request.Bucket, request.Key, "") + response, err := c.do(req) + if err != nil { + return err + } + return consumeOSSResponse(response) +} + +func (c *OSSHTTPClient) newRequest(ctx context.Context, method, endpoint, bucket, key, subresource string, body io.Reader) (*http.Request, error) { + requestURL, err := ossObjectURL(endpoint, bucket, key, subresource) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body) + if err != nil { + return nil, errors.New("build OSS request") + } + return req, nil +} + +func (c *OSSHTTPClient) sign(req *http.Request, bucket, key, subresource string) { + req.Header.Set("Date", c.now().UTC().Format(http.TimeFormat)) + canonicalResource := "/" + bucket + "/" + key + if subresource != "" { + canonicalResource += "?" + subresource + } + stringToSign := strings.Join([]string{ + req.Method, + req.Header.Get("Content-MD5"), + req.Header.Get("Content-Type"), + req.Header.Get("Date"), + canonicalOSSHeaders(req.Header) + canonicalResource, + }, "\n") + mac := hmac.New(sha1.New, []byte(c.accessKeySecret)) + _, _ = mac.Write([]byte(stringToSign)) + signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) + req.Header.Set("Authorization", "OSS "+c.accessKeyID+":"+signature) +} + +func (c *OSSHTTPClient) do(req *http.Request) (*http.Response, error) { + response, err := c.client.Do(req) + if err == nil { + return response, nil + } + if ctxErr := req.Context().Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, &OSSError{Err: errors.New("OSS transport request failed")} +} + +func ossObjectURL(endpoint, bucket, key, subresource string) (*url.URL, error) { + endpoint = strings.TrimSpace(endpoint) + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + u, err := url.Parse(endpoint) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, errors.New("invalid OSS endpoint") + } + if bucket == "" || strings.ContainsAny(bucket, "/\\") || key == "" || strings.HasPrefix(key, "/") { + return nil, errors.New("invalid OSS bucket or object key") + } + + basePath := strings.TrimRight(u.Path, "/") + pathStyle := basePath != "" && path.Base(basePath) == bucket + hostHasBucket := strings.HasPrefix(strings.ToLower(u.Hostname()), strings.ToLower(bucket)+".") + if !pathStyle && !hostHasBucket { + hostname := bucket + "." + u.Hostname() + if port := u.Port(); port != "" { + hostname += ":" + port + } + u.Host = hostname + } + u.Path = basePath + "/" + key + u.RawPath = strings.TrimRight(escapedURLPath(basePath), "/") + "/" + escapeOSSKey(key) + if subresource != "" { + u.RawQuery = subresource + } + return u, nil +} + +func escapedURLPath(value string) string { + if value == "" { + return "" + } + parts := strings.Split(value, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + return strings.Join(parts, "/") +} + +func canonicalOSSHeaders(header http.Header) string { + keys := make([]string, 0) + values := make(map[string]string) + for key, entries := range header { + lowerKey := strings.ToLower(key) + if !strings.HasPrefix(lowerKey, "x-oss-") { + continue + } + keys = append(keys, lowerKey) + trimmed := make([]string, len(entries)) + for i, entry := range entries { + trimmed[i] = strings.TrimSpace(entry) + } + values[lowerKey] = strings.Join(trimmed, ",") + } + sort.Strings(keys) + var canonical strings.Builder + for _, key := range keys { + fmt.Fprintf(&canonical, "%s:%s\n", key, values[key]) + } + return canonical.String() +} + +func consumeOSSResponse(response *http.Response) error { + if successfulOSSStatus(response.StatusCode) { + _, _ = io.Copy(io.Discard, response.Body) + return response.Body.Close() + } + return readOSSError(response) +} + +func successfulOSSStatus(status int) bool { return status >= 200 && status < 300 } + +func readOSSError(response *http.Response) error { + defer response.Body.Close() + var wire struct { + Code string `xml:"Code"` + } + _ = xml.NewDecoder(io.LimitReader(response.Body, maxOSSErrorBody)).Decode(&wire) + return &OSSError{Status: response.StatusCode, Code: strings.TrimSpace(wire.Code)} +} + +var _ OSSClient = (*OSSHTTPClient)(nil) diff --git a/backend/internal/assets/oss_http_test.go b/backend/internal/assets/oss_http_test.go new file mode 100644 index 0000000..329f32d --- /dev/null +++ b/backend/internal/assets/oss_http_test.go @@ -0,0 +1,196 @@ +package assets + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" +) + +type ossRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f ossRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestOSSHTTPClientSignsPutAndBuildsVirtualHostURL(t *testing.T) { + client := newTestOSSHTTPClient(t, func(r *http.Request) *http.Response { + if r.Method != http.MethodPut || r.URL.String() != "https://bucket-a.oss-cn-test.aliyuncs.com/photos/cat.png" { + t.Fatalf("request = %s %s", r.Method, r.URL) + } + if r.Host != "bucket-a.oss-cn-test.aliyuncs.com" || r.Header.Get("Date") != "Thu, 13 Aug 2026 00:00:00 GMT" { + t.Fatalf("host/date = %q / %q", r.Host, r.Header.Get("Date")) + } + if r.Header.Get("Content-Type") != "image/png" || r.Header.Get("Content-Length") != "3" || r.ContentLength != 3 { + t.Fatalf("content type/length = %q / %q / %d", r.Header.Get("Content-Type"), r.Header.Get("Content-Length"), r.ContentLength) + } + if got := r.Header.Get("Authorization"); got != "OSS access-key:ZBQxu+qcAZzglIi4GbBMhAIZIYs=" { + t.Fatalf("Authorization = %q", got) + } + body, _ := io.ReadAll(r.Body) + if string(body) != "png" { + t.Fatalf("body = %q", body) + } + return ossHTTPResponse(http.StatusOK, "", nil) + }) + + err := client.Put(context.Background(), OSSPutRequest{ + Endpoint: "oss-cn-test.aliyuncs.com/", Bucket: "bucket-a", Key: "photos/cat.png", + ContentType: "image/png", Body: strings.NewReader("png"), Size: 3, + }) + if err != nil { + t.Fatal(err) + } +} + +func TestOSSHTTPClientSignsACLGetDeleteAndStreamsGetMetadata(t *testing.T) { + var calls int + client := newTestOSSHTTPClient(t, func(r *http.Request) *http.Response { + calls++ + if r.URL.Host != "bucket-a.oss-cn-test.aliyuncs.com" || r.URL.Path != "/photos/cat.png" { + t.Fatalf("URL = %s", r.URL) + } + switch calls { + case 1: + if r.Method != http.MethodPut || r.URL.RawQuery != "acl" || r.Header.Get("x-oss-object-acl") != OSSACLPublicRead { + t.Fatalf("ACL request = %s %s headers=%v", r.Method, r.URL, r.Header) + } + if got := r.Header.Get("Authorization"); got != "OSS access-key:B8m2FqdnBSJTA4F6L3o7NZkkYTU=" { + t.Fatalf("ACL Authorization = %q", got) + } + return ossHTTPResponse(http.StatusOK, "", nil) + case 2: + if r.Method != http.MethodGet || r.Header.Get("Authorization") != "OSS access-key:+do4MvonQT0wF7/Pwl+7+Eqze9g=" { + t.Fatalf("GET request = %s auth=%q", r.Method, r.Header.Get("Authorization")) + } + return ossHTTPResponse(http.StatusOK, "image/png", []byte("streamed")) + case 3: + if r.Method != http.MethodDelete || r.Header.Get("Authorization") != "OSS access-key:ykyNN5k2axmVkPpEqzNsEVHCmGU=" { + t.Fatalf("DELETE request = %s auth=%q", r.Method, r.Header.Get("Authorization")) + } + return ossHTTPResponse(http.StatusNoContent, "", nil) + default: + t.Fatalf("unexpected call %d", calls) + return nil + } + }) + + ctx := context.Background() + object := OSSObjectRequest{Endpoint: "https://bucket-a.oss-cn-test.aliyuncs.com", Bucket: "bucket-a", Key: "photos/cat.png"} + if err := client.SetACL(ctx, OSSACLRequest{Endpoint: object.Endpoint, Bucket: object.Bucket, Key: object.Key, ACL: OSSACLPublicRead}); err != nil { + t.Fatal(err) + } + blob, err := client.Get(ctx, object) + if err != nil { + t.Fatal(err) + } + if blob.ContentType != "image/png" || blob.Size != 8 { + t.Fatalf("blob metadata = %#v", blob) + } + got, _ := io.ReadAll(blob.Body) + _ = blob.Body.Close() + if string(got) != "streamed" { + t.Fatalf("body = %q", got) + } + if err := client.Delete(ctx, object); err != nil { + t.Fatal(err) + } +} + +func TestOSSHTTPClientUsesPathStyleForExplicitBucketPathAndEscapesKey(t *testing.T) { + client := newTestOSSHTTPClient(t, func(r *http.Request) *http.Response { + if got := r.URL.String(); got != "https://proxy.test/storage/bucket-a/a%20b/%E7%8C%AB.png" { + t.Fatalf("URL = %q", got) + } + return ossHTTPResponse(http.StatusNoContent, "", nil) + }) + if err := client.Delete(context.Background(), OSSObjectRequest{ + Endpoint: "https://proxy.test/storage/bucket-a", Bucket: "bucket-a", Key: "a b/猫.png", + }); err != nil { + t.Fatal(err) + } +} + +func TestOSSHTTPClientMapsServiceErrorsWithoutLeakingResponseOrCredentials(t *testing.T) { + client := newTestOSSHTTPClient(t, func(*http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusNotFound, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader( + `NoSuchKeyaccess-key secret-key private-object`, + )), + } + }) + _, err := client.Get(context.Background(), OSSObjectRequest{Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "bucket-a", Key: "private-object"}) + var ossErr *OSSError + if !errors.As(err, &ossErr) || ossErr.Status != http.StatusNotFound || ossErr.Code != "NoSuchKey" { + t.Fatalf("error = %#v", err) + } + for _, secret := range []string{"access-key", "secret-key", "private-object"} { + if strings.Contains(err.Error(), secret) { + t.Fatalf("error leaks %q: %v", secret, err) + } + } +} + +func TestOSSHTTPClientPropagatesContextCancellation(t *testing.T) { + client := newTestOSSHTTPClient(t, func(r *http.Request) *http.Response { + <-r.Context().Done() + return nil + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := client.Delete(ctx, OSSObjectRequest{Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "bucket-a", Key: "object"}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context cancellation", err) + } +} + +func TestNewOSSHTTPClientRejectsMissingDependencies(t *testing.T) { + for _, test := range []struct { + id, secret string + client *http.Client + }{ + {"", "secret", http.DefaultClient}, + {"id", "", http.DefaultClient}, + {"id", "secret", nil}, + } { + if _, err := NewOSSHTTPClient(test.id, test.secret, test.client, time.Now); err == nil { + t.Fatalf("NewOSSHTTPClient(%q, %q, %v) succeeded", test.id, test.secret, test.client) + } + } +} + +func newTestOSSHTTPClient(t *testing.T, roundTrip func(*http.Request) *http.Response) *OSSHTTPClient { + t.Helper() + httpClient := &http.Client{Transport: ossRoundTripFunc(func(r *http.Request) (*http.Response, error) { + response := roundTrip(r) + if response == nil { + return nil, r.Context().Err() + } + response.Request = r + return response, nil + })} + client, err := NewOSSHTTPClient("access-key", "secret-key", httpClient, func() time.Time { + return time.Date(2026, 8, 13, 0, 0, 0, 0, time.UTC) + }) + if err != nil { + t.Fatal(err) + } + return client +} + +func ossHTTPResponse(status int, contentType string, body []byte) *http.Response { + header := make(http.Header) + if contentType != "" { + header.Set("Content-Type", contentType) + } + return &http.Response{ + StatusCode: status, + Header: header, + Body: io.NopCloser(bytes.NewReader(body)), + ContentLength: int64(len(body)), + } +} diff --git a/backend/internal/assets/oss_test.go b/backend/internal/assets/oss_test.go new file mode 100644 index 0000000..8445c43 --- /dev/null +++ b/backend/internal/assets/oss_test.go @@ -0,0 +1,95 @@ +package assets + +import ( + "bytes" + "context" + "errors" + "io" + "testing" +) + +type ossClientStub struct { + puts []OSSPutRequest + acls []OSSACLRequest + gets []OSSObjectRequest + deletes []OSSObjectRequest + body []byte + contentType string + getErr, errorToReturn error +} + +func (s *ossClientStub) Put(_ context.Context, r OSSPutRequest) error { + s.puts = append(s.puts, r) + _, _ = io.ReadAll(r.Body) + return s.errorToReturn +} +func (s *ossClientStub) SetACL(_ context.Context, r OSSACLRequest) error { + s.acls = append(s.acls, r) + return s.errorToReturn +} +func (s *ossClientStub) Get(_ context.Context, r OSSObjectRequest) (Blob, error) { + s.gets = append(s.gets, r) + if s.getErr != nil { + return Blob{}, s.getErr + } + return Blob{Body: io.NopCloser(bytes.NewReader(s.body)), ContentType: s.contentType, Size: int64(len(s.body))}, nil +} +func (s *ossClientStub) Delete(_ context.Context, r OSSObjectRequest) error { + s.deletes = append(s.deletes, r) + return s.errorToReturn +} + +func TestOSSPutUsesConfiguredAddressContentTypeAndPublicACL(t *testing.T) { + client := &ossClientStub{} + store, err := NewOSS(OSSConfig{Endpoint: "https://oss-cn.test", Bucket: "bucket-a", PublicBaseURL: "https://cdn.test/root", PublicRead: true}, client) + if err != nil { + t.Fatal(err) + } + got, err := store.Put(context.Background(), "uploads/a b.png", bytes.NewReader([]byte("png")), 3, "image/png") + if err != nil { + t.Fatal(err) + } + if got.Key != "uploads/a b.png" || got.URL != "https://cdn.test/root/uploads/a%20b.png" { + t.Fatalf("stored=%#v", got) + } + if len(client.puts) != 1 || client.puts[0].Endpoint != "https://oss-cn.test" || client.puts[0].Bucket != "bucket-a" || client.puts[0].ContentType != "image/png" || client.puts[0].Size != 3 { + t.Fatalf("puts=%#v", client.puts) + } + if len(client.acls) != 1 || client.acls[0].ACL != OSSACLPublicRead { + t.Fatalf("acls=%#v", client.acls) + } +} +func TestOSSReadDeleteAndMissingMapping(t *testing.T) { + client := &ossClientStub{body: []byte("x"), contentType: "image/png"} + store, _ := NewOSS(OSSConfig{Endpoint: "e", Bucket: "b", PublicBaseURL: "https://cdn.test"}, client) + blob, err := store.Read(context.Background(), "generated-results/x.png") + if err != nil { + t.Fatal(err) + } + _ = blob.Body.Close() + if len(client.gets) != 1 || client.gets[0].Key != "generated-results/x.png" { + t.Fatalf("gets=%#v", client.gets) + } + if err = store.Delete(context.Background(), "generated-results/x.png"); err != nil || len(client.deletes) != 1 { + t.Fatalf("delete=%v %#v", err, client.deletes) + } + client.getErr = &OSSError{Status: 404, Code: "NoSuchKey", Err: errors.New("secret response")} + if _, err = store.Read(context.Background(), "missing"); !errors.Is(err, ErrBlobNotFound) { + t.Fatalf("missing=%v", err) + } + client.errorToReturn = &OSSError{Status: 404, Code: "NoSuchKey"} + if err = store.Delete(context.Background(), "missing"); err != nil { + t.Fatalf("delete missing=%v", err) + } +} +func TestOSSRejectsUnsafeKeysAndIncompleteConfiguration(t *testing.T) { + if _, err := NewOSS(OSSConfig{Endpoint: "e", Bucket: "b"}, &ossClientStub{}); err == nil { + t.Fatal("expected config error") + } + store, _ := NewOSS(OSSConfig{Endpoint: "e", Bucket: "b", PublicBaseURL: "https://cdn.test"}, &ossClientStub{}) + for _, key := range []string{"", "../secret", "/absolute", "a\\b"} { + if _, err := store.Put(context.Background(), key, bytes.NewReader(nil), 0, "x"); !errors.Is(err, ErrUnsafeBlobKey) { + t.Errorf("key %q: %v", key, err) + } + } +} diff --git a/backend/internal/assets/remote.go b/backend/internal/assets/remote.go index ee295c9..bc8dcf5 100644 --- a/backend/internal/assets/remote.go +++ b/backend/internal/assets/remote.go @@ -10,9 +10,40 @@ import ( "net/url" ) +var errRemotePolicyRequired = errors.New("remote fetcher requires a bounded client and destination policy") + type HTTPRemoteFetcher struct { client *http.Client maxBytes int64 + policy DestinationPolicy +} + +// DestinationPolicy is the injectable outbound-network policy seam. +type DestinationPolicy interface { + Validate(context.Context, *url.URL) error +} + +// NewPolicyHTTPRemoteFetcher preserves the existing constructor while offering +// production composition a fail-closed destination-policy seam. +func NewPolicyHTTPRemoteFetcher(client *http.Client, maxBytes int64, policy DestinationPolicy) (*HTTPRemoteFetcher, error) { + if client == nil || client.Timeout <= 0 || policy == nil { + return nil, errRemotePolicyRequired + } + base := *client + previousRedirect := base.CheckRedirect + base.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + if err := policy.Validate(req.Context(), req.URL); err != nil { + return err + } + if previousRedirect != nil { + return previousRedirect(req, via) + } + return nil + } + return &HTTPRemoteFetcher{client: &base, maxBytes: maxBytes, policy: policy}, nil } func NewHTTPRemoteFetcher(client *http.Client, maxBytes int64) *HTTPRemoteFetcher { @@ -40,6 +71,14 @@ func (f *HTTPRemoteFetcher) Fetch(ctx context.Context, rawURL string) (Blob, err if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return Blob{}, ErrRemoteProtocol } + if u.User != nil { + return Blob{}, ErrRemoteProtocol + } + if f.policy != nil { + if err := f.policy.Validate(ctx, u); err != nil { + return Blob{}, err + } + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return Blob{}, err diff --git a/backend/internal/assets/remote_policy.go b/backend/internal/assets/remote_policy.go new file mode 100644 index 0000000..b95e171 --- /dev/null +++ b/backend/internal/assets/remote_policy.go @@ -0,0 +1,28 @@ +package assets + +import ( + "net/http" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/webhook" +) + +// PublicDestinationPolicy reuses the backend's public-network validation and +// DNS-pinned dialing policy for provider asset downloads. +type PublicDestinationPolicy = webhook.PublicDestinationValidator + +func NewPublicDestinationPolicy(resolver webhook.IPResolver, dialer webhook.ContextDialer) *PublicDestinationPolicy { + return webhook.NewPublicDestinationPolicy(resolver, dialer) +} + +// NewPublicHTTPRemoteFetcher is the production construction seam. Its +// transport resolves and validates immediately before dialing the selected IP. +func NewPublicHTTPRemoteFetcher(timeout time.Duration, maxBytes int64, policy *PublicDestinationPolicy) (*HTTPRemoteFetcher, error) { + if policy == nil { + return nil, errRemotePolicyRequired + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.DialContext = policy.DialContext + return NewPolicyHTTPRemoteFetcher(&http.Client{Timeout: timeout, Transport: transport}, maxBytes, policy) +} diff --git a/backend/internal/assets/remote_test.go b/backend/internal/assets/remote_test.go index b542191..ef0bcd2 100644 --- a/backend/internal/assets/remote_test.go +++ b/backend/internal/assets/remote_test.go @@ -6,9 +6,24 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" + "strings" "testing" + "time" ) +type assetRoundTripFunc func(*http.Request) (*http.Response, error) + +func (function assetRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return function(request) +} + +type remotePolicyFunc func(context.Context, *url.URL) error + +func (function remotePolicyFunc) Validate(ctx context.Context, target *url.URL) error { + return function(ctx, target) +} + func TestHTTPRemoteFetcherRestrictsProtocolAndSize(t *testing.T) { fetcher := NewHTTPRemoteFetcher(http.DefaultClient, 4) if _, err := fetcher.Fetch(context.Background(), "file:///etc/passwd"); !errors.Is(err, ErrRemoteProtocol) { @@ -40,3 +55,37 @@ func TestHTTPRemoteFetcherReturnsBoundedBody(t *testing.T) { t.Fatalf("blob = %#v body=%q", blob, body) } } + +func TestPolicyHTTPRemoteFetcherValidatesInitialAndRedirectDestinations(t *testing.T) { + validated := make([]string, 0, 2) + policy := remotePolicyFunc(func(_ context.Context, target *url.URL) error { + validated = append(validated, target.Hostname()) + if target.Hostname() == "private.test" { + return errors.New("private destination") + } + return nil + }) + client := &http.Client{Timeout: time.Second, Transport: assetRoundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.URL.Hostname() == "public.test" { + return &http.Response{StatusCode: http.StatusFound, Header: http.Header{"Location": []string{"http://private.test/asset"}}, Body: io.NopCloser(strings.NewReader("")), Request: request}, nil + } + t.Fatal("redirect target transport must not run") + return nil, nil + })} + fetcher, err := NewPolicyHTTPRemoteFetcher(client, 4, policy) + if err != nil { + t.Fatal(err) + } + if _, err := fetcher.Fetch(context.Background(), "https://public.test/asset"); err == nil { + t.Fatal("policy-denied redirect was accepted") + } + if strings.Join(validated, ",") != "public.test,private.test" { + t.Fatalf("validated destinations = %v", validated) + } +} + +func TestNewPublicHTTPRemoteFetcherRequiresPolicy(t *testing.T) { + if _, err := NewPublicHTTPRemoteFetcher(time.Second, 4, nil); err == nil { + t.Fatal("nil public destination policy accepted") + } +} diff --git a/backend/internal/assets/service_test.go b/backend/internal/assets/service_test.go index 2d8053a..7e39515 100644 --- a/backend/internal/assets/service_test.go +++ b/backend/internal/assets/service_test.go @@ -74,6 +74,58 @@ func TestUploadCompensatesBlobWhenCatalogCreateFails(t *testing.T) { } } +func TestImportGeneratedDownloadsStoresAndPersistsGeneratedMetadata(t *testing.T) { + cat := &memoryCatalog{} + blobs := &memoryBlobs{} + remote := &memoryRemote{blob: Blob{Body: io.NopCloser(bytes.NewReader([]byte("png"))), ContentType: "image/png", Size: 3}} + ids := []string{"file-1", "asset-1"} + svc := NewService(cat, blobs, remote, func() time.Time { return time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) }, func(string) string { + id := ids[0] + ids = ids[1:] + return id + }) + created, err := svc.ImportGenerated(context.Background(), PlatformScope("owner-a"), ImportGeneratedCommand{URL: "https://provider.test/output.png?token=x", Capability: "image.generate", JobID: "job-1", Kind: KindImage, Tags: []string{"generated", "job:job-1"}}) + if err != nil { + t.Fatal(err) + } + if created.Source != SourceGenerated || created.StoragePath != "generated-results/2026-08-13/file-1-output.png" || created.Metadata["importedFrom"] == nil || created.Metadata["jobId"] != "job-1" || !remote.called { + t.Fatalf("created = %#v", created) + } +} + +func TestImportGeneratedCompensatesBlobWhenCatalogFails(t *testing.T) { + cat := &memoryCatalog{createErr: errors.New("database unavailable")} + blobs := &memoryBlobs{} + remote := &memoryRemote{blob: Blob{Body: io.NopCloser(bytes.NewReader([]byte("video"))), ContentType: "video/mp4", Size: 5}} + svc := NewService(cat, blobs, remote, time.Now, func(prefix string) string { return prefix + "-1" }) + _, err := svc.ImportGenerated(context.Background(), PlatformScope("owner-a"), ImportGeneratedCommand{URL: "https://provider.test/output.mp4", Capability: "video.generate", JobID: "job-1"}) + if !errors.Is(err, cat.createErr) || len(blobs.deleted) != 1 || blobs.deleted[0] != blobs.putKey { + t.Fatalf("error=%v deleted=%#v key=%q", err, blobs.deleted, blobs.putKey) + } +} + +func TestImportMockStoresAccessibleGeneratedAssetWithoutRemoteFetch(t *testing.T) { + cat := &memoryCatalog{} + blobs := &memoryBlobs{} + remote := &memoryRemote{} + ids := []string{"file-1", "asset-1"} + svc := NewService(cat, blobs, remote, func() time.Time { return time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) }, func(string) string { + id := ids[0] + ids = ids[1:] + return id + }) + created, err := svc.ImportMock(context.Background(), PlatformScope("owner-a"), ImportMockCommand{Capability: "image.generate", JobID: "job-1", Kind: KindImage, Tags: []string{"generated", "job:job-1"}}) + if err != nil { + t.Fatal(err) + } + if created.ID != "asset-1" || created.StoragePath != "generated-results/2026-08-13/file-1-mock-image.svg" || created.URL != "https://app.test/generated-results/2026-08-13/file-1-mock-image.svg" || created.Metadata["mock"] != true || remote.called { + t.Fatalf("created = %#v remoteCalled=%v", created, remote.called) + } + if blobs.putContentType != "image/svg+xml" || !bytes.Contains(blobs.putBody, []byte("job-1")) { + t.Fatalf("stored contentType=%q body=%q", blobs.putContentType, blobs.putBody) + } +} + func TestDeleteLeavesCatalogWhenBlobDeletionFails(t *testing.T) { cat := &memoryCatalog{assets: []Asset{{ID: "a", OwnerID: "o", StoragePath: "uploads/a"}}} blobs := &memoryBlobs{deleteErr: errors.New("storage down")} @@ -167,14 +219,18 @@ func (m *memoryCatalog) DeleteOwner(_ context.Context, owner, id string) (Asset, } type memoryBlobs struct { - putKey string - deleted []string - read Blob - deleteErr error + putKey string + putBody []byte + putContentType string + deleted []string + read Blob + deleteErr error } -func (m *memoryBlobs) Put(_ context.Context, key string, _ io.Reader, _ int64, _ string) (StoredObject, error) { +func (m *memoryBlobs) Put(_ context.Context, key string, body io.Reader, _ int64, contentType string) (StoredObject, error) { m.putKey = key + m.putBody, _ = io.ReadAll(body) + m.putContentType = contentType return StoredObject{Key: key, URL: "https://app.test/" + key}, nil } func (m *memoryBlobs) Read(context.Context, string) (Blob, error) { return m.read, nil } diff --git a/backend/internal/billing/catalog.go b/backend/internal/billing/catalog.go index 157128d..97ae9eb 100644 --- a/backend/internal/billing/catalog.go +++ b/backend/internal/billing/catalog.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "sort" + "strconv" "strings" ) @@ -41,27 +42,39 @@ type ConditionRange struct { } type ParameterTier struct { - Value any - Match any - StandardFactor, MarkupMultiplier float64 - Enabled bool + Value any `json:"value"` + Label string `json:"label,omitempty"` + Match any `json:"match,omitempty"` + StandardFactor float64 `json:"standardFactor"` + MarkupMultiplier float64 `json:"markupMultiplier"` + Enabled bool `json:"enabled"` + Note string `json:"note,omitempty"` } type ParameterDimension struct { - Key string - DefaultValue any - BaselineValue any - Tiers []ParameterTier + Key string `json:"key"` + Label string `json:"label,omitempty"` + DefaultValue any `json:"defaultValue,omitempty"` + BaselineValue any `json:"baselineValue"` + Tiers []ParameterTier `json:"tiers"` } type PriceRule struct { - ID, Provider, Capability, ReqKey string - Unit Unit - QuantitySource QuantitySource - StandardUnitPriceFen int64 - MarkupMultiplier float64 - Enabled bool - Conditions Conditions - Priority int - Dimensions []ParameterDimension + ID string `json:"id"` + Provider string `json:"provider"` + Capability string `json:"capability"` + ReqKey string `json:"reqKey,omitempty"` + VariantKey string `json:"variantKey,omitempty"` + Unit Unit `json:"unit"` + QuantitySource QuantitySource `json:"quantitySource,omitempty"` + StandardUnitPriceFen int64 `json:"standardUnitPriceFen"` + MarkupMultiplier float64 `json:"markupMultiplier"` + Enabled bool `json:"enabled"` + Conditions Conditions `json:"conditions,omitempty"` + Priority int `json:"priority,omitempty"` + Note string `json:"note,omitempty"` + Source map[string]any `json:"source,omitempty"` + Dimensions []ParameterDimension `json:"parameterDimensions,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + UpdatedAt string `json:"updatedAt,omitempty"` } type QuoteInput struct { Provider, Capability, ReqKey, Source, Role string @@ -69,13 +82,23 @@ type QuoteInput struct { BillingDisabled bool } type Quote struct { - PriceRuleID string - Unit Unit - Quantity float64 - StandardUnitPriceFen, AmountFen int64 - MarkupMultiplier float64 - Currency string - QuotaExempt bool + PriceRuleID string `json:"priceRuleId"` + Provider string `json:"provider,omitempty"` + Capability string `json:"capability,omitempty"` + ReqKey string `json:"reqKey,omitempty"` + VariantKey string `json:"variantKey,omitempty"` + Unit Unit `json:"unit"` + Quantity float64 `json:"quantity"` + StandardUnitPriceFen int64 `json:"standardUnitPriceFen"` + AmountFen int64 `json:"amountFen"` + MarkupMultiplier float64 `json:"markupMultiplier"` + Currency string `json:"currency"` + Conditions Conditions `json:"conditions,omitempty"` + QuantitySource QuantitySource `json:"quantitySource,omitempty"` + Parameters Parameters `json:"parameters,omitempty"` + QuotaExempt bool `json:"quotaExempt,omitempty"` + ReservedAmountFen int64 `json:"reservedAmountFen,omitempty"` + SettlementStatus string `json:"settlementStatus,omitempty"` } type Catalog struct{ Rules []PriceRule } @@ -89,14 +112,16 @@ func (c Catalog) Quote(input QuoteInput) (*Quote, error) { } var matches []candidate for _, rule := range c.Rules { - if !rule.Enabled || rule.Provider != input.Provider || rule.Capability != input.Capability || rule.ReqKey != "" && rule.ReqKey != input.ReqKey || !conditionsMatch(rule.Conditions, input.Parameters) { + conditions := effectiveConditions(rule) + if !rule.Enabled || rule.Provider != input.Provider || rule.Capability != input.Capability || rule.ReqKey != "" && rule.ReqKey != input.ReqKey || !conditionsMatch(conditions, input.Parameters) { continue } req := 0 if rule.ReqKey != "" { req = 1 } - matches = append(matches, candidate{rule, req, len(rule.Conditions), rule.Priority}) + rule.Conditions = conditions + matches = append(matches, candidate{rule, req, len(conditions), rule.Priority}) } if len(matches) == 0 { return nil, ErrPriceRuleNotFound @@ -123,7 +148,40 @@ func (c Catalog) Quote(input QuoteInput) (*Quote, error) { return nil, err } quantity := quantityFor(w.rule, input.Parameters) - return &Quote{PriceRuleID: w.rule.ID, Unit: w.rule.Unit, Quantity: quantity, StandardUnitPriceFen: price, MarkupMultiplier: markup, AmountFen: int64(math.Ceil(float64(price) * quantity * markup)), Currency: CurrencyCNY, QuotaExempt: input.Source == "platform" && input.Role == "super_admin"}, nil + return &Quote{PriceRuleID: w.rule.ID, Provider: input.Provider, Capability: input.Capability, ReqKey: input.ReqKey, VariantKey: w.rule.VariantKey, Unit: w.rule.Unit, Quantity: quantity, StandardUnitPriceFen: price, MarkupMultiplier: markup, AmountFen: int64(math.Ceil(float64(price) * quantity * markup)), Currency: CurrencyCNY, Conditions: w.rule.Conditions, QuantitySource: w.rule.QuantitySource, Parameters: input.Parameters, QuotaExempt: input.Source == "platform" && input.Role == "super_admin"}, nil +} + +func effectiveConditions(rule PriceRule) Conditions { + conditions := parseVariantKey(rule.VariantKey) + for key, value := range rule.Conditions { + conditions[key] = value + } + return conditions +} + +func parseVariantKey(value string) Conditions { + conditions := Conditions{} + for _, part := range strings.FieldsFunc(value, func(r rune) bool { return r == ';' || r == ',' }) { + key, raw, ok := strings.Cut(part, "=") + if !ok { + continue + } + key, raw = strings.TrimSpace(key), strings.TrimSpace(raw) + if key == "ratio" { + key = "aspectRatio" + } + switch key { + case "model", "resolution", "size", "aspectRatio", "quality": + if raw != "" { + conditions[key] = strings.ToLower(raw) + } + case "duration", "imageCount", "referenceImageCount", "scale": + if number, err := strconv.ParseFloat(raw, 64); err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) { + conditions[key] = number + } + } + } + return conditions } func tierPrice(rule PriceRule, params Parameters) (int64, float64, error) { diff --git a/backend/internal/billing/catalog_test.go b/backend/internal/billing/catalog_test.go index 8b64c3c..d287de5 100644 --- a/backend/internal/billing/catalog_test.go +++ b/backend/internal/billing/catalog_test.go @@ -76,3 +76,31 @@ func TestCatalogExemptsDisabledMockAndSuperAdmin(t *testing.T) { t.Fatalf("quote = %#v, %v", quote, err) } } + +func TestCatalogMatchesLegacyVariantKeyAsEffectiveConditions(t *testing.T) { + rules := []PriceRule{ + {ID: "720", Provider: "seedance", Capability: "video.generate", ReqKey: "seedance-2", VariantKey: "resolution=720p", Unit: UnitVideoSecond, StandardUnitPriceFen: 99, MarkupMultiplier: 1.2, Enabled: true}, + {ID: "1080", Provider: "seedance", Capability: "video.generate", ReqKey: "seedance-2", VariantKey: "resolution=1080p", Unit: UnitVideoSecond, StandardUnitPriceFen: 248, MarkupMultiplier: 1.2, Enabled: true}, + } + quote, err := (Catalog{Rules: rules}).Quote(QuoteInput{ + Provider: "seedance", Capability: "video.generate", ReqKey: "seedance-2", + Parameters: Parameters{"resolution": " 720P ", "duration": 5.0}, + }) + if err != nil { + t.Fatal(err) + } + if quote.PriceRuleID != "720" || quote.VariantKey != "resolution=720p" || quote.AmountFen != 594 { + t.Fatalf("quote = %#v", quote) + } + if quote.Conditions["resolution"] != "720p" { + t.Fatalf("effective conditions = %#v", quote.Conditions) + } +} + +func TestCatalogExplicitConditionsOverrideLegacyVariantKey(t *testing.T) { + rule := PriceRule{ID: "override", Provider: "seedance", Capability: "video.generate", VariantKey: "resolution=720p", Conditions: Conditions{"resolution": "1080p"}, Unit: UnitRequest, StandardUnitPriceFen: 1, MarkupMultiplier: 1, Enabled: true} + quote, err := (Catalog{Rules: []PriceRule{rule}}).Quote(QuoteInput{Provider: "seedance", Capability: "video.generate", Parameters: Parameters{"resolution": "1080P"}}) + if err != nil || quote == nil || quote.Conditions["resolution"] != "1080p" { + t.Fatalf("quote = %#v, err = %v", quote, err) + } +} diff --git a/backend/internal/billing/commit.go b/backend/internal/billing/commit.go new file mode 100644 index 0000000..b34737c --- /dev/null +++ b/backend/internal/billing/commit.go @@ -0,0 +1,8 @@ +package billing + +import "errors" + +// ErrCommitOutcomeUnknown means a database commit response was lost and the +// durable billing outcome could not be reconciled. Callers must not write a +// compensating failure state because the charge may already have committed. +var ErrCommitOutcomeUnknown = errors.New("billing commit outcome unknown") diff --git a/backend/internal/billing/defaults.go b/backend/internal/billing/defaults.go new file mode 100644 index 0000000..3b67437 --- /dev/null +++ b/backend/internal/billing/defaults.go @@ -0,0 +1,177 @@ +package billing + +import ( + "context" + "fmt" + "math" + "strings" +) + +const ( + DefaultBillingMarkupMultiplier = 1.2 + BillingCatalogObservedAt = "2026-08-11" +) + +// PriceRuleSeeder is an optional store capability. Existing Store +// implementations remain source-compatible and seed-capable stores can make +// the official base catalog durable before quotes are read. +type PriceRuleSeeder interface { + SeedBillingPriceRules(context.Context, []PriceRule) error +} + +func DefaultBillingPriceRules() []PriceRule { + rules := []PriceRule{ + {ID: "base-volcengine-jimeng-seedream46", Provider: "volcengine-visual", Capability: "image.generate", ReqKey: "jimeng_seedream46_cvtob", Unit: UnitImage, StandardUnitPriceFen: 20, Note: "即梦4.6公开资源包基准。", Source: catalogSource("https://www.volcengine.com/activity/jimeng")}, + {ID: "base-evolink-gpt-image-2", Provider: "evolink", Capability: "image.generate", ReqKey: "gpt-image-2", Unit: UnitImage, StandardUnitPriceFen: 34, Note: "medium / 1K / 1:1 / 无参考图基准。", Source: catalogSource("https://evolink.ai/zh/gpt-image-2"), Dimensions: evolinkDimensions()}, + {ID: "base-bailian-wan27-image-pro", Provider: "bailian", Capability: "image.generate", ReqKey: "wan2.7-image-pro", Unit: UnitImage, StandardUnitPriceFen: 50, Source: catalogSource("https://help.aliyun.com/zh/model-studio/wan2-7-image-pro")}, + {ID: "base-bailian-wan27-i2v-720p", Provider: "bailian", Capability: "video.generate", ReqKey: "wan2.7-i2v-2026-04-25", VariantKey: "resolution=720p", Unit: UnitVideoSecond, StandardUnitPriceFen: 60, Source: catalogSource("https://help.aliyun.com/zh/model-studio/wan2-7-i2v")}, + {ID: "base-bailian-wan27-i2v-1080p", Provider: "bailian", Capability: "video.generate", ReqKey: "wan2.7-i2v-2026-04-25", VariantKey: "resolution=1080p", Unit: UnitVideoSecond, StandardUnitPriceFen: 100, Source: catalogSource("https://help.aliyun.com/zh/model-studio/wan2-7-i2v")}, + {ID: "base-seedance-2-0-480p", Provider: "seedance", Capability: "video.generate", ReqKey: "doubao-seedance-2-0-260128", VariantKey: "resolution=480p", Unit: UnitVideoSecond, StandardUnitPriceFen: 46, Source: catalogSource("https://www.volcengine.com/docs/82379/1544106?lang=zh")}, + {ID: "base-seedance-2-0-720p", Provider: "seedance", Capability: "video.generate", ReqKey: "doubao-seedance-2-0-260128", VariantKey: "resolution=720p", Unit: UnitVideoSecond, StandardUnitPriceFen: 99, Source: catalogSource("https://www.volcengine.com/docs/82379/1544106?lang=zh")}, + {ID: "base-seedance-2-0-1080p", Provider: "seedance", Capability: "video.generate", ReqKey: "doubao-seedance-2-0-260128", VariantKey: "resolution=1080p", Unit: UnitVideoSecond, StandardUnitPriceFen: 248, Source: catalogSource("https://www.volcengine.com/docs/82379/1544106?lang=zh")}, + {ID: "base-seedance-2-0-4k", Provider: "seedance", Capability: "video.generate", ReqKey: "doubao-seedance-2-0-260128", VariantKey: "resolution=4k", Unit: UnitVideoSecond, StandardUnitPriceFen: 505, Source: catalogSource("https://www.volcengine.com/docs/82379/1544106?lang=zh")}, + } + for i := range rules { + rules[i].Enabled = true + rules[i].MarkupMultiplier = DefaultBillingMarkupMultiplier + } + return rules +} + +func catalogSource(url string) map[string]any { + return map[string]any{"url": url, "observedAt": BillingCatalogObservedAt} +} + +func evolinkDimensions() []ParameterDimension { + tiers := func(values ...any) []ParameterTier { + out := make([]ParameterTier, 0, len(values)) + for index := 0; index < len(values); index += 2 { + out = append(out, ParameterTier{Value: values[index], StandardFactor: values[index+1].(float64), MarkupMultiplier: DefaultBillingMarkupMultiplier, Enabled: true}) + } + return out + } + return []ParameterDimension{ + {Key: "quality", BaselineValue: "medium", DefaultValue: "medium", Tiers: tiers("low", .11, "medium", 1.0, "high", 4.0)}, + {Key: "resolution", BaselineValue: "1K", DefaultValue: "1K", Tiers: tiers("1K", 1.0, "2K", 4.0, "4K", 8.0)}, + {Key: "aspectRatio", BaselineValue: "1:1", DefaultValue: "1:1", Tiers: tiers("1:1", 1.0, "4:3", 1.0, "16:9", 1.0, "9:16", 1.0)}, + {Key: "referenceImageCount", BaselineValue: 0, DefaultValue: 0, Tiers: []ParameterTier{ + {Value: 0, StandardFactor: 1, MarkupMultiplier: DefaultBillingMarkupMultiplier, Enabled: true}, + {Value: "1–4", Match: map[string]any{"min": 1, "max": 4}, StandardFactor: 1, MarkupMultiplier: DefaultBillingMarkupMultiplier, Enabled: true}, + {Value: "5–8", Match: map[string]any{"min": 5, "max": 8}, StandardFactor: 1, MarkupMultiplier: DefaultBillingMarkupMultiplier, Enabled: true}, + {Value: "9–16", Match: map[string]any{"min": 9, "max": 16}, StandardFactor: 1, MarkupMultiplier: DefaultBillingMarkupMultiplier, Enabled: true}, + }}, + } +} + +// NormalizeBillingParameters derives quote inputs from the persisted generation +// payload. Explicit parameters are retained only as a fallback for values the +// payload does not carry. +func NormalizeBillingParameters(payload map[string]any, fallback Parameters) Parameters { + out := Parameters{} + for key, value := range fallback { + out[key] = value + } + settings := record(payload["settings"]) + providerPayload := record(payload["providerPayload"]) + providerParameters := record(providerPayload["parameters"]) + input := record(payload["input"]) + inputSettings := record(input["settings"]) + + setText(out, "model", first(providerPayload["model"], input["model"], settings["model"])) + setText(out, "resolution", first(settings["resolution"], providerParameters["resolution"], providerPayload["resolution"], inputSettings["resolution"], input["resolution"])) + setText(out, "quality", first(input["quality"], settings["quality"], providerPayload["quality"], providerParameters["quality"])) + setNumber(out, "duration", first(settings["duration"], providerParameters["duration"], providerPayload["duration"], inputSettings["duration"], input["duration"]), false) + setNumber(out, "imageCount", first(providerPayload["n"], providerParameters["n"], input["n"], input["imageCount"]), true) + setNumber(out, "scale", first(input["scale"], settings["scale"]), false) + + width, widthOK := finiteNumber(first(providerPayload["width"], input["width"])) + height, heightOK := finiteNumber(first(providerPayload["height"], input["height"])) + if size := normalizedText(first(providerPayload["size"], providerParameters["size"], inputSettings["size"], input["size"])); size != "" { + out["size"] = strings.ReplaceAll(size, "×", "*") + } else if widthOK && heightOK { + out["size"] = fmt.Sprintf("%g*%g", width, height) + } + if ratio := normalizedText(first(settings["ratio"], settings["aspectRatio"], providerPayload["ratio"], providerParameters["ratio"], inputSettings["ratio"], input["ratio"], input["aspectRatio"])); ratio != "" { + out["aspectRatio"] = ratio + } else if widthOK && heightOK && height > 0 { + out["aspectRatio"] = aspectRatio(width, height) + } + + if count := referenceCount(payload, input, providerPayload); count > 0 { + out["referenceImageCount"] = float64(count) + } + if value, ok := boolean(first(settings["generate_audio"], settings["generateAudio"], providerParameters["generate_audio"], providerParameters["generateAudio"], input["generate_audio"], input["generateAudio"])); ok { + out["generateAudio"] = value + } + return out +} + +func record(value any) map[string]any { result, _ := value.(map[string]any); return result } +func first(values ...any) any { + for _, value := range values { + if value != nil { + return value + } + } + return nil +} +func normalizedText(value any) string { + if value == nil { + return "" + } + return strings.ToLower(strings.TrimSpace(fmt.Sprint(value))) +} +func setText(out Parameters, key string, value any) { + if value != nil { + if text := normalizedText(value); text != "" { + out[key] = text + } + } +} +func setNumber(out Parameters, key string, value any, ceil bool) { + if number, ok := finiteNumber(value); ok && (!ceil || number > 0) { + if ceil { + number = math.Ceil(number) + } + out[key] = number + } +} +func finiteNumber(value any) (float64, bool) { return number(value) } +func boolean(value any) (bool, bool) { + switch v := value.(type) { + case bool: + return v, true + case string: + if strings.EqualFold(strings.TrimSpace(v), "true") { + return true, true + } + if strings.EqualFold(strings.TrimSpace(v), "false") { + return false, true + } + } + return false, false +} +func aspectRatio(width, height float64) string { + ratio := width / height + for _, item := range []struct { + value float64 + label string + }{{1, "1:1"}, {4.0 / 3, "4:3"}, {16.0 / 9, "16:9"}, {3.0 / 4, "3:4"}, {9.0 / 16, "9:16"}, {21.0 / 9, "21:9"}} { + if math.Abs(ratio-item.value) < .02 { + return item.label + } + } + return fmt.Sprintf("%g:%g", width, height) +} +func referenceCount(payload, input, providerPayload map[string]any) int { + count := 0 + for _, value := range []any{payload["imageUrls"], payload["inputUrls"], input["imageUrls"], providerPayload["image_urls"], providerPayload["imageUrls"]} { + switch items := value.(type) { + case []any: + count += len(items) + case []string: + count += len(items) + } + } + return count +} diff --git a/backend/internal/billing/defaults_test.go b/backend/internal/billing/defaults_test.go new file mode 100644 index 0000000..2007609 --- /dev/null +++ b/backend/internal/billing/defaults_test.go @@ -0,0 +1,32 @@ +package billing + +import "testing" + +func TestDefaultBillingPriceRulesFreezeOfficialCatalogMatchKeys(t *testing.T) { + rules := DefaultBillingPriceRules() + want := map[string]bool{ + "base-volcengine-jimeng-seedream46": false, + "base-evolink-gpt-image-2": false, + "base-bailian-wan27-image-pro": false, + "base-bailian-wan27-i2v-720p": false, + "base-bailian-wan27-i2v-1080p": false, + "base-seedance-2-0-480p": false, + "base-seedance-2-0-720p": false, + "base-seedance-2-0-1080p": false, + "base-seedance-2-0-4k": false, + } + for _, rule := range rules { + if _, ok := want[rule.ID]; !ok { + t.Fatalf("unexpected default rule %q", rule.ID) + } + want[rule.ID] = true + if !rule.Enabled || rule.MarkupMultiplier != 1.2 || rule.StandardUnitPriceFen <= 0 { + t.Fatalf("invalid default rule %#v", rule) + } + } + for id, found := range want { + if !found { + t.Errorf("missing default rule %q", id) + } + } +} diff --git a/backend/internal/billing/http_service.go b/backend/internal/billing/http_service.go new file mode 100644 index 0000000..f9451e3 --- /dev/null +++ b/backend/internal/billing/http_service.go @@ -0,0 +1,156 @@ +package billing + +import ( + "context" + "errors" + "fmt" + "math" + "strings" + "time" +) + +type Wallet struct { + OrganizationID string `json:"organizationId"` + BalanceFen int64 `json:"balanceFen"` + TotalRechargedFen int64 `json:"totalRechargedFen"` + TotalChargedFen int64 `json:"totalChargedFen"` + Currency string `json:"currency,omitempty"` + UpdatedAt string `json:"updatedAt"` +} + +type LedgerEntry struct { + ID string `json:"id"` + OrganizationID string `json:"organizationId"` + AccountID string `json:"accountId,omitempty"` + JobID string `json:"jobId,omitempty"` + Kind string `json:"kind"` + DeltaFen int64 `json:"deltaFen"` + BalanceAfterFen int64 `json:"balanceAfterFen"` + Currency string `json:"currency"` + IdempotencyKey string `json:"idempotencyKey"` + Description string `json:"description"` + Metadata map[string]any `json:"metadata"` + CreatedAt string `json:"createdAt"` +} + +type Summary struct { + RechargeFen int64 `json:"rechargeFen"` + ChargedFen int64 `json:"chargedFen"` + RefundedFen int64 `json:"refundedFen"` + NetConsumedFen int64 `json:"netConsumedFen"` +} +type AccountConfig struct { + AccountName string `json:"accountName,omitempty"` + BankName string `json:"bankName,omitempty"` + AccountNumber string `json:"accountNumber,omitempty"` + Contact string `json:"contact,omitempty"` +} +type Organization struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + ArchiveOwnerID string `json:"archiveOwnerId"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Wallet Wallet `json:"wallet"` +} +type Member struct { + ID string `json:"id"` + DisplayName string `json:"displayName"` + Phone string `json:"phone"` + Role string `json:"role"` + OrganizationID string `json:"organizationId,omitempty"` + Status string `json:"status"` +} +type Overview struct { + Wallet Wallet `json:"wallet"` + Ledger []LedgerEntry `json:"ledger"` + BillingAccount AccountConfig `json:"billingAccount"` + Summary Summary `json:"summary"` + Personal Summary `json:"personal"` +} +type AdminOverview struct { + BillingAccount AccountConfig `json:"billingAccount"` + Organizations []Organization `json:"organizations"` + Members []Member `json:"members"` + Ledger []LedgerEntry `json:"ledger"` + PriceRules []PriceRule `json:"priceRules"` +} + +type QuoteCommand struct { + AccountID, OrganizationID, OrganizationName, Role string + Provider, Capability, ReqKey string + Parameters Parameters + Payload map[string]any +} +type PricePatch struct { + MarkupMultiplier float64 + DimensionKey, TierValue string +} +type AdjustmentCommand struct { + OrganizationID, OperatorID, Direction, Note string + AmountFen, DeltaFen int64 +} +type AdjustmentResult struct { + Wallet Wallet + Entry LedgerEntry +} + +type AccountConfigStore interface { + Load(context.Context) (AccountConfig, error) + Save(context.Context, AccountConfig) error +} + +// ReadService is the deliberately cohesive persistence seam required by the +// Billing HTTP module. Implementations can execute overview reads concurrently. +type ReadService interface { + Overview(context.Context, string, string) (Overview, error) + AdminOverview(context.Context) (AdminOverview, error) + ListPrices(context.Context) ([]PriceRule, error) + GetPrice(context.Context, string) (*PriceRule, error) +} +type HTTPService interface { + ReadService + Quote(context.Context, QuoteCommand) (*Quote, error) + UpdatePrice(context.Context, string, PricePatch) (*PriceRule, error) + Adjust(context.Context, AdjustmentCommand) (AdjustmentResult, error) +} + +var ErrPriceNotFound = errors.New("billing price not found") + +func ValidatePricePatch(rule *PriceRule, patch PricePatch) error { + if patch.MarkupMultiplier < 1 || patch.MarkupMultiplier > 1000 || math.IsNaN(patch.MarkupMultiplier) || math.IsInf(patch.MarkupMultiplier, 0) { + return errors.New("上浮倍率必须在 1.00 至 1000.00 之间。") + } + if (patch.DimensionKey == "") != (patch.TierValue == "") { + return errors.New("参数档位倍率更新必须同时提供参数维度和档位。") + } + if rule == nil { + return ErrPriceNotFound + } + if patch.DimensionKey == "" && len(rule.Dimensions) > 0 { + return errors.New("当前服务包含参数档位,请指定要调整的参数档位倍率。") + } + if patch.DimensionKey != "" { + for _, dimension := range rule.Dimensions { + if dimension.Key != patch.DimensionKey { + continue + } + for _, tier := range dimension.Tiers { + if strings.TrimSpace(fmt.Sprint(tier.Value)) == patch.TierValue { + return nil + } + } + return errors.New("计费参数档位不存在。") + } + return errors.New("计费参数维度不存在。") + } + return nil +} + +func PostingResult(post WalletPosting, organizationID string) AdjustmentResult { + return AdjustmentResult{ + Wallet: Wallet{OrganizationID: organizationID, BalanceFen: post.BalanceFen, TotalRechargedFen: post.TotalRechargedFen, TotalChargedFen: post.TotalChargedFen, Currency: CurrencyCNY, UpdatedAt: post.UpdatedAt.Format(time.RFC3339Nano)}, + Entry: LedgerEntry{ID: post.LedgerID, OrganizationID: organizationID, DeltaFen: post.DeltaFen, BalanceAfterFen: post.BalanceAfterFen, Currency: CurrencyCNY, CreatedAt: post.CreatedAt.Format(time.RFC3339Nano)}, + } +} diff --git a/backend/internal/billing/service.go b/backend/internal/billing/service.go new file mode 100644 index 0000000..3c40aee --- /dev/null +++ b/backend/internal/billing/service.go @@ -0,0 +1,201 @@ +package billing + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" +) + +type Store interface { + BillingWallet(context.Context, string) (Wallet, error) + BillingWallets(context.Context) ([]Wallet, error) + BillingLedger(context.Context, string, string, int) ([]LedgerEntry, error) + BillingOrganizations(context.Context) ([]Organization, error) + BillingMembers(context.Context) ([]Member, error) + BillingOrganizationExists(context.Context, string) (bool, error) + ListBillingPriceRules(context.Context, bool) ([]PriceRule, error) + GetBillingPriceRule(context.Context, string) (*PriceRule, error) + UpdateBillingPriceRule(context.Context, string, PricePatch) (*PriceRule, error) + PostBillingWalletEntry(context.Context, WalletPostParams) (WalletPosting, error) +} + +type Service struct { + store Store + newID func() string + enabled bool +} + +func NewService(store Store, newID func() string) *Service { + if newID == nil { + newID = billingID + } + return &Service{store: store, newID: newID, enabled: true} +} + +// SetEnabled configures whether generation quotes are required. The default is +// enabled; disabling preserves the historical optional-billing behavior and +// does not consult or seed the price catalog. +func (s *Service) SetEnabled(enabled bool) *Service { + s.enabled = enabled + return s +} + +func (s *Service) Overview(ctx context.Context, organizationID, accountID string) (Overview, error) { + wallet, err := s.store.BillingWallet(ctx, organizationID) + if err != nil { + return Overview{}, err + } + ledger, err := s.store.BillingLedger(ctx, organizationID, "", 500) + if err != nil { + return Overview{}, err + } + personal, err := s.store.BillingLedger(ctx, organizationID, accountID, 500) + if err != nil { + return Overview{}, err + } + return Overview{Wallet: wallet, Ledger: ledger, Summary: Summarize(ledger), Personal: Summarize(personal)}, nil +} +func (s *Service) Quote(ctx context.Context, command QuoteCommand) (*Quote, error) { + if !s.enabled { + return nil, nil + } + if seeder, ok := s.store.(PriceRuleSeeder); ok { + if err := seeder.SeedBillingPriceRules(ctx, DefaultBillingPriceRules()); err != nil { + return nil, err + } + } + rules, err := s.store.ListBillingPriceRules(ctx, false) + if err != nil { + return nil, err + } + parameters := NormalizeBillingParameters(command.Payload, command.Parameters) + quote, err := (Catalog{Rules: rules}).Quote(QuoteInput{Provider: command.Provider, Capability: command.Capability, ReqKey: command.ReqKey, Source: "platform", Role: command.Role, Parameters: parameters}) + if err != nil || quote == nil || command.Provider != "seedance" || command.ReqKey != "doubao-seedance-2-0-260128" { + return quote, err + } + inputVideo, inputDuration := seedanceInputVideo(command.Payload) + estimated, err := EstimateSeedanceAmountFen(SeedanceEstimateInput{ + Resolution: fmt.Sprint(parameters["resolution"]), AspectRatio: fmt.Sprint(parameters["aspectRatio"]), + OutputDurationSeconds: quote.Quantity, InputVideo: inputVideo, InputVideoDurationSeconds: inputDuration, + MarkupMultiplier: quote.MarkupMultiplier, + }) + if err != nil { + return nil, err + } + if estimated > quote.AmountFen { + quote.AmountFen = estimated + } + quote.ReservedAmountFen = quote.AmountFen + quote.SettlementStatus = "pending" + if quote.Parameters == nil { + quote.Parameters = Parameters{} + } + quote.Parameters["inputVideo"] = inputVideo + return quote, nil +} +func (s *Service) AdminOverview(ctx context.Context) (AdminOverview, error) { + organizations, err := s.store.BillingOrganizations(ctx) + if err != nil { + return AdminOverview{}, err + } + wallets, err := s.store.BillingWallets(ctx) + if err != nil { + return AdminOverview{}, err + } + members, err := s.store.BillingMembers(ctx) + if err != nil { + return AdminOverview{}, err + } + ledger, err := s.store.BillingLedger(ctx, "", "", 500) + if err != nil { + return AdminOverview{}, err + } + rules, err := s.store.ListBillingPriceRules(ctx, true) + if err != nil { + return AdminOverview{}, err + } + byOrganization := map[string]Wallet{} + for _, wallet := range wallets { + byOrganization[wallet.OrganizationID] = wallet + } + for i := range organizations { + if wallet, ok := byOrganization[organizations[i].ID]; ok { + organizations[i].Wallet = wallet + } else { + organizations[i].Wallet = Wallet{OrganizationID: organizations[i].ID, Currency: CurrencyCNY, UpdatedAt: organizations[i].UpdatedAt} + } + } + return AdminOverview{Organizations: organizations, Members: members, Ledger: ledger, PriceRules: rules}, nil +} +func (s *Service) ListPrices(ctx context.Context) ([]PriceRule, error) { + return s.store.ListBillingPriceRules(ctx, true) +} +func (s *Service) GetPrice(ctx context.Context, id string) (*PriceRule, error) { + return s.store.GetBillingPriceRule(ctx, id) +} +func (s *Service) UpdatePrice(ctx context.Context, id string, patch PricePatch) (*PriceRule, error) { + return s.store.UpdateBillingPriceRule(ctx, id, patch) +} +func (s *Service) Adjust(ctx context.Context, command AdjustmentCommand) (AdjustmentResult, error) { + exists, err := s.store.BillingOrganizationExists(ctx, command.OrganizationID) + if err != nil { + return AdjustmentResult{}, err + } + if !exists { + return AdjustmentResult{}, &StatusError{400, errors.New("组织不存在。")} + } + kind, description := "adjustment", "管理员扣减 · "+command.Note + if command.Direction == "credit" { + kind, description = "recharge", "管理员上账 · "+command.Note + } + post, err := s.store.PostBillingWalletEntry(ctx, WalletPostParams{LedgerID: s.newID(), OrganizationID: command.OrganizationID, Kind: kind, DeltaFen: command.DeltaFen, Currency: CurrencyCNY, IdempotencyKey: "manual-adjustment:" + s.newID(), Description: description, Metadata: map[string]any{"operation": map[bool]string{true: "admin_top_up", false: "manual_adjustment"}[command.Direction == "credit"], "direction": command.Direction, "note": command.Note, "operatorId": command.OperatorID, "amountYuan": fmt.Sprintf("%.2f", float64(command.AmountFen)/100)}}) + if err != nil { + message := err.Error() + if strings.Contains(message, "BILLING_INSUFFICIENT_BALANCE") { + return AdjustmentResult{}, &StatusError{402, ErrInsufficientBalance} + } + if strings.Contains(message, "BILLING_IDEMPOTENCY_PAYLOAD_MISMATCH") { + return AdjustmentResult{}, &StatusError{409, ErrIdempotencyConflict} + } + return AdjustmentResult{}, err + } + return PostingResult(post, command.OrganizationID), nil +} + +func Summarize(entries []LedgerEntry) Summary { + var out Summary + for _, entry := range entries { + switch { + case entry.Kind == "recharge", entry.Kind == "adjustment" && entry.DeltaFen > 0: + if entry.DeltaFen > 0 { + out.RechargeFen += entry.DeltaFen + } + case entry.Kind == "charge": + if entry.DeltaFen < 0 { + out.ChargedFen -= entry.DeltaFen + } + case entry.Kind == "refund": + if entry.DeltaFen > 0 { + out.RefundedFen += entry.DeltaFen + } + } + } + out.NetConsumedFen = out.ChargedFen - out.RefundedFen + if out.NetConsumedFen < 0 { + out.NetConsumedFen = 0 + } + return out +} +func billingID() string { + raw := make([]byte, 12) + if _, err := rand.Read(raw); err != nil { + return fmt.Sprintf("entry-%d", time.Now().UnixNano()) + } + return "entry-" + hex.EncodeToString(raw) +} + +var _ HTTPService = (*Service)(nil) diff --git a/backend/internal/billing/service_test.go b/backend/internal/billing/service_test.go new file mode 100644 index 0000000..1fc6b8e --- /dev/null +++ b/backend/internal/billing/service_test.go @@ -0,0 +1,135 @@ +package billing + +import ( + "context" + "reflect" + "testing" +) + +func TestSummarizeUsesFenAndDoesNotProduceNegativeConsumption(t *testing.T) { + got := Summarize([]LedgerEntry{{Kind: "recharge", DeltaFen: 1000}, {Kind: "adjustment", DeltaFen: 100}, {Kind: "charge", DeltaFen: -400}, {Kind: "refund", DeltaFen: 500}}) + if got.RechargeFen != 1100 || got.ChargedFen != 400 || got.RefundedFen != 500 || got.NetConsumedFen != 0 { + t.Fatalf("summary=%+v", got) + } +} + +func TestNormalizeBillingParametersUsesGenerationPayloadShapes(t *testing.T) { + payload := map[string]any{ + "settings": map[string]any{"resolution": " 1080P ", "ratio": "16:9", "duration": 5.2}, + "providerPayload": map[string]any{"parameters": map[string]any{"n": 1.2}, "width": 1024.0, "height": 768.0}, + "imageUrls": []any{"a", "b"}, + } + got := NormalizeBillingParameters(payload, Parameters{"resolution": "480p", "callerOnly": true}) + want := Parameters{"resolution": "1080p", "size": "1024*768", "aspectRatio": "16:9", "duration": 5.2, "imageCount": 2.0, "referenceImageCount": 2.0, "callerOnly": true} + if !reflect.DeepEqual(got, want) { + t.Fatalf("parameters = %#v, want %#v", got, want) + } +} + +func TestServiceQuoteNormalizesPayloadAndSeedsDefaultsThroughOptionalCapability(t *testing.T) { + store := "eStoreStub{rules: []PriceRule{{ID: "720", Provider: "seedance", Capability: "video.generate", ReqKey: "seedance-2", VariantKey: "resolution=720p", Unit: UnitVideoSecond, StandardUnitPriceFen: 99, MarkupMultiplier: 1.2, Enabled: true}}} + quote, err := NewService(store, nil).Quote(context.Background(), QuoteCommand{ + Provider: "seedance", Capability: "video.generate", ReqKey: "seedance-2", + Parameters: Parameters{"resolution": "1080p"}, + Payload: map[string]any{"settings": map[string]any{"resolution": "720P", "duration": 5.0}}, + }) + if err != nil { + t.Fatal(err) + } + if store.seedCalls != 1 || len(store.seeded) == 0 { + t.Fatalf("seed calls = %d, rules = %d", store.seedCalls, len(store.seeded)) + } + if quote.PriceRuleID != "720" || quote.Quantity != 5 || quote.Parameters["resolution"] != "720p" { + t.Fatalf("quote = %#v", quote) + } +} + +func TestServiceQuoteDisabledReturnsNilWithoutAccessingRules(t *testing.T) { + store := "eStoreStub{failOnAccess: true} + quote, err := NewService(store, nil).SetEnabled(false).Quote(context.Background(), QuoteCommand{Provider: "seedance", Capability: "video.generate"}) + if err != nil || quote != nil { + t.Fatalf("quote = %#v, err = %v", quote, err) + } + if store.seedCalls != 0 || store.listCalls != 0 { + t.Fatalf("seed calls = %d, list calls = %d", store.seedCalls, store.listCalls) + } +} + +func TestServiceQuoteFreezesConservativeSeedanceReserve(t *testing.T) { + store := "eStoreStub{rules: []PriceRule{{ + ID: "seedance-720", Provider: "seedance", Capability: "video.generate", + ReqKey: "doubao-seedance-2-0-260128", VariantKey: "resolution=720p", + Unit: UnitVideoSecond, StandardUnitPriceFen: 99, MarkupMultiplier: 1.2, Enabled: true, + }}} + quote, err := NewService(store, nil).Quote(context.Background(), QuoteCommand{ + Provider: "seedance", Capability: "video.generate", ReqKey: "doubao-seedance-2-0-260128", + Payload: map[string]any{ + "settings": map[string]any{"duration": 5.0, "resolution": "720p", "ratio": "9:16"}, + "promptAssembly": map[string]any{"materials": []any{map[string]any{ + "type": "video", "url": "https://example.test/reference.mp4", + }}}, + }, + }) + if err != nil { + t.Fatal(err) + } + if quote.AmountFen != 1452 || quote.ReservedAmountFen != 1452 || quote.SettlementStatus != "pending" || quote.Parameters["inputVideo"] != true { + t.Fatalf("quote = %#v", quote) + } +} + +func TestSeedanceEstimateMatchesExecutableTypeScriptVectors(t *testing.T) { + withoutVideo, err := EstimateSeedanceAmountFen(SeedanceEstimateInput{ + Resolution: "720p", AspectRatio: "16:9", OutputDurationSeconds: 5, MarkupMultiplier: 1.2, + }) + if err != nil || withoutVideo != 597 { + t.Fatalf("without video = %d, %v", withoutVideo, err) + } + withUnknownVideo, err := EstimateSeedanceAmountFen(SeedanceEstimateInput{ + Resolution: "720p", AspectRatio: "9:16", OutputDurationSeconds: 5, + InputVideo: true, MarkupMultiplier: 1.2, + }) + if err != nil || withUnknownVideo != 1452 { + t.Fatalf("with video = %d, %v", withUnknownVideo, err) + } +} + +type quoteStoreStub struct { + rules []PriceRule + seeded []PriceRule + seedCalls int + listCalls int + failOnAccess bool +} + +func (s *quoteStoreStub) SeedBillingPriceRules(_ context.Context, rules []PriceRule) error { + s.seedCalls++ + s.seeded = rules + return nil +} +func (s *quoteStoreStub) ListBillingPriceRules(context.Context, bool) ([]PriceRule, error) { + s.listCalls++ + if s.failOnAccess { + panic("disabled billing accessed price rules") + } + return s.rules, nil +} +func (*quoteStoreStub) BillingWallet(context.Context, string) (Wallet, error) { return Wallet{}, nil } +func (*quoteStoreStub) BillingWallets(context.Context) ([]Wallet, error) { return nil, nil } +func (*quoteStoreStub) BillingLedger(context.Context, string, string, int) ([]LedgerEntry, error) { + return nil, nil +} +func (*quoteStoreStub) BillingOrganizations(context.Context) ([]Organization, error) { return nil, nil } +func (*quoteStoreStub) BillingMembers(context.Context) ([]Member, error) { return nil, nil } +func (*quoteStoreStub) BillingOrganizationExists(context.Context, string) (bool, error) { + return false, nil +} +func (*quoteStoreStub) GetBillingPriceRule(context.Context, string) (*PriceRule, error) { + return nil, nil +} +func (*quoteStoreStub) UpdateBillingPriceRule(context.Context, string, PricePatch) (*PriceRule, error) { + return nil, nil +} +func (*quoteStoreStub) PostBillingWalletEntry(context.Context, WalletPostParams) (WalletPosting, error) { + return WalletPosting{}, nil +} diff --git a/backend/internal/billing/settlement.go b/backend/internal/billing/settlement.go new file mode 100644 index 0000000..49f7ea3 --- /dev/null +++ b/backend/internal/billing/settlement.go @@ -0,0 +1,192 @@ +package billing + +import ( + "context" + "errors" + "fmt" + "math" + "strconv" + "strings" +) + +const seedanceTokenScale = 1_000_000 + +const ( + seedanceFPS = 24 + seedanceMaximumInputDuration = 15 +) + +type SeedanceEstimateInput struct { + Resolution, AspectRatio string + OutputDurationSeconds float64 + InputVideoDurationSeconds float64 + InputVideo bool + MarkupMultiplier float64 +} + +type SeedanceActualAmountInput struct { + Resolution string + InputVideo bool + CompletionTokens int64 + MarkupMultiplier float64 +} + +func SeedanceTokenPriceFenPerMillion(resolution string, inputVideo bool) int64 { + normalized := strings.ToLower(strings.TrimSpace(resolution)) + var withoutVideo, withVideo int64 + switch normalized { + case "480p", "720p": + withoutVideo, withVideo = 4600, 2800 + case "1080p": + withoutVideo, withVideo = 5100, 3100 + case "4k": + withoutVideo, withVideo = 2600, 1600 + default: + withoutVideo, withVideo = 4600, 2800 + } + if inputVideo { + return withVideo + } + return withoutVideo +} + +func CalculateSeedanceActualAmountFen(input SeedanceActualAmountInput) (int64, error) { + if input.CompletionTokens <= 0 { + return 0, errors.New("seedance completion tokens must be positive") + } + if math.IsNaN(input.MarkupMultiplier) || math.IsInf(input.MarkupMultiplier, 0) || input.MarkupMultiplier < 1 { + return 0, errors.New("seedance markup multiplier must be at least 1") + } + amount := math.Ceil(float64(input.CompletionTokens) * float64(SeedanceTokenPriceFenPerMillion(input.Resolution, input.InputVideo)) * input.MarkupMultiplier / seedanceTokenScale) + if amount < 1 { + amount = 1 + } + return int64(amount), nil +} + +func EstimateSeedanceAmountFen(input SeedanceEstimateInput) (int64, error) { + if math.IsNaN(input.OutputDurationSeconds) || math.IsInf(input.OutputDurationSeconds, 0) || input.OutputDurationSeconds <= 0 { + return 0, errors.New("seedance output duration must be positive") + } + if math.IsNaN(input.MarkupMultiplier) || math.IsInf(input.MarkupMultiplier, 0) || input.MarkupMultiplier < 1 { + return 0, errors.New("seedance markup multiplier must be at least 1") + } + inputDuration := 0.0 + if input.InputVideo { + inputDuration = input.InputVideoDurationSeconds + if math.IsNaN(inputDuration) || math.IsInf(inputDuration, 0) || inputDuration <= 0 { + inputDuration = seedanceMaximumInputDuration + } + inputDuration = math.Min(seedanceMaximumInputDuration, inputDuration) + } + width, height := seedanceOutputDimensions(input.Resolution, input.AspectRatio) + tokens := math.Ceil((inputDuration + input.OutputDurationSeconds) * float64(width*height*seedanceFPS) / 1024) + amount := math.Ceil(tokens * float64(SeedanceTokenPriceFenPerMillion(input.Resolution, input.InputVideo)) * input.MarkupMultiplier / seedanceTokenScale) + return int64(math.Max(1, amount)), nil +} + +func seedanceOutputDimensions(resolution, aspectRatio string) (int, int) { + baseWidth, baseHeight := 1280, 720 + switch strings.ToLower(strings.TrimSpace(resolution)) { + case "480p": + baseWidth, baseHeight = 854, 480 + case "1080p": + baseWidth, baseHeight = 1920, 1080 + case "4k": + baseWidth, baseHeight = 3840, 2160 + } + ratio := 16.0 / 9.0 + parts := strings.FieldsFunc(strings.TrimSpace(aspectRatio), func(r rune) bool { return r == ':' || r == '/' }) + if len(parts) == 2 { + if width, widthErr := strconv.ParseFloat(parts[0], 64); widthErr == nil && width > 0 { + if height, heightErr := strconv.ParseFloat(parts[1], 64); heightErr == nil && height > 0 { + ratio = width / height + } + } + } + area := float64(baseWidth * baseHeight) + return max(1, int(math.Round(math.Sqrt(area*ratio)))), max(1, int(math.Round(math.Sqrt(area/ratio)))) +} + +func seedanceInputVideo(payload map[string]any) (bool, float64) { + materials := billingMaterials(payload) + total, known := 0.0, false + for _, material := range materials { + if !strings.EqualFold(strings.TrimSpace(fmt.Sprint(material["type"])), "video") { + continue + } + for _, value := range []any{material["duration"], material["durationSeconds"], record(material["metadata"])["duration"], record(material["metadata"])["durationSeconds"]} { + if duration, ok := number(value); ok && duration > 0 { + total += duration + known = true + break + } + } + } + if len(materials) == 0 { + return false, 0 + } + inputVideo := false + for _, material := range materials { + if strings.EqualFold(strings.TrimSpace(fmt.Sprint(material["type"])), "video") { + inputVideo = true + break + } + } + if !known { + return inputVideo, 0 + } + return inputVideo, math.Min(seedanceMaximumInputDuration, total) +} + +func billingMaterials(payload map[string]any) []map[string]any { + for _, candidate := range []any{record(payload["assembled"])["materials"], record(payload["promptAssembly"])["materials"], record(payload["input"])["materials"], payload["materials"]} { + values, ok := candidate.([]any) + if !ok || len(values) == 0 { + continue + } + out := make([]map[string]any, 0, len(values)) + for _, value := range values { + if material := record(value); len(material) > 0 { + out = append(out, material) + } + } + if len(out) > 0 { + return out + } + } + return nil +} + +// SettlementRequest expresses actual usage minus the amount already charged. +// A positive DeltaFen is an additional charge; a negative value is a refund. +type SettlementRequest struct { + OrganizationID, AccountID, JobID, Description string + DeltaFen int64 + Metadata map[string]any +} + +func (l Ledger) Settle(ctx context.Context, input SettlementRequest) (WalletPosting, error) { + if input.DeltaFen == 0 { + return WalletPosting{}, errors.New("billing settlement delta must be non-zero") + } + kind, walletDelta := "charge", -input.DeltaFen + if input.DeltaFen < 0 { + kind = "refund" + } + return l.post(ctx, ChargeRequest{ + OrganizationID: input.OrganizationID, + AccountID: input.AccountID, + JobID: input.JobID, + AmountFen: absInt64(input.DeltaFen), + Description: input.Description, + Metadata: input.Metadata, + }, kind, walletDelta, "job-settlement:"+input.JobID) +} + +func absInt64(value int64) int64 { + if value < 0 { + return -value + } + return value +} diff --git a/backend/internal/billing/settlement_test.go b/backend/internal/billing/settlement_test.go new file mode 100644 index 0000000..79f64e7 --- /dev/null +++ b/backend/internal/billing/settlement_test.go @@ -0,0 +1,50 @@ +package billing + +import ( + "context" + "testing" + "time" +) + +func TestSeedanceActualAmountUsesFrozenTokenPricingAndRoundsUp(t *testing.T) { + tests := []struct { + resolution string + inputVideo bool + wantPrice int64 + wantAmount int64 + }{ + {resolution: "720P", wantPrice: 4600, wantAmount: 6}, + {resolution: "1080p", inputVideo: true, wantPrice: 3100, wantAmount: 4}, + {resolution: "4K", wantPrice: 2600, wantAmount: 4}, + } + for _, test := range tests { + if got := SeedanceTokenPriceFenPerMillion(test.resolution, test.inputVideo); got != test.wantPrice { + t.Fatalf("price(%q, %v) = %d, want %d", test.resolution, test.inputVideo, got, test.wantPrice) + } + got, err := CalculateSeedanceActualAmountFen(SeedanceActualAmountInput{Resolution: test.resolution, InputVideo: test.inputVideo, CompletionTokens: 1001, MarkupMultiplier: 1.2}) + if err != nil || got != test.wantAmount { + t.Fatalf("amount(%q, %v) = %d, %v; want %d", test.resolution, test.inputVideo, got, err, test.wantAmount) + } + } +} + +func TestLedgerSettlesSeedanceDeltaWithOneFrozenIdempotencyKey(t *testing.T) { + poster := &recordingPoster{result: WalletPosting{LedgerID: "settlement-ledger", CreatedAt: time.Unix(7, 0)}} + ledger := Ledger{Poster: poster, NewID: func() string { return "new-ledger" }} + + posting, err := ledger.Settle(context.Background(), SettlementRequest{ + OrganizationID: "org-1", AccountID: "account-1", JobID: "job-1", + DeltaFen: 25, Description: "actual usage charge", Metadata: map[string]any{"operation": "seedance_actual_settlement"}, + }) + if err != nil || posting.LedgerID != "settlement-ledger" { + t.Fatalf("posting = %#v, err = %v", posting, err) + } + if poster.params.Kind != "charge" || poster.params.DeltaFen != -25 || poster.params.IdempotencyKey != "job-settlement:job-1" { + t.Fatalf("charge params = %#v", poster.params) + } + + _, err = ledger.Settle(context.Background(), SettlementRequest{OrganizationID: "org-1", JobID: "job-2", DeltaFen: -9}) + if err != nil || poster.params.Kind != "refund" || poster.params.DeltaFen != 9 || poster.params.IdempotencyKey != "job-settlement:job-2" { + t.Fatalf("refund params = %#v, err = %v", poster.params, err) + } +} diff --git a/backend/internal/httpapi/admin.go b/backend/internal/httpapi/admin.go new file mode 100644 index 0000000..b0c6eae --- /dev/null +++ b/backend/internal/httpapi/admin.go @@ -0,0 +1,288 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" +) + +type adminHandler struct { + authorizer *PlatformAuthorizer + service *administration.Service +} + +func NewAdminHandler(authorizer *PlatformAuthorizer, service *administration.Service) (http.Handler, error) { + if authorizer == nil || service == nil { + return nil, fmt.Errorf("admin HTTP: authorizer and service are required") + } + return &adminHandler{authorizer: authorizer, service: service}, nil +} + +func (handler *adminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/admin/accounts/groups": + handler.serveGroups(w, r) + case "/api/admin/accounts/password": + handler.servePasswordReset(w, r) + case "/api/admin/accounts": + handler.serveAccounts(w, r) + case "/api/admin/organizations": + handler.serveOrganizations(w, r) + default: + http.NotFound(w, r) + } +} + +func (handler *adminHandler) authorize(w http.ResponseWriter, r *http.Request) (identity.Session, administration.Actor, bool) { + session, err := handler.authorizer.Authorize(r, PlatformAdmin) + if err != nil { + writeAdminError(w, err) + return identity.Session{}, administration.Actor{}, false + } + return session, administration.Actor{ID: session.User.ID, Role: administration.Role(session.User.Role), OrganizationID: session.User.OrganizationID}, true +} + +func (handler *adminHandler) serveGroups(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + adminMethodNotAllowed(w, http.MethodPost) + return + } + writeAdminJSON(w, http.StatusGone, map[string]any{"error": "平台账号体系不再使用外部部门接口,请直接管理组织。"}) +} + +func (handler *adminHandler) serveAccounts(w http.ResponseWriter, r *http.Request) { + session, actor, ok := handler.authorize(w, r) + if !ok { + return + } + switch r.Method { + case http.MethodGet: + organizationID := strings.TrimSpace(r.URL.Query().Get("organizationId")) + if actor.Role != administration.RoleSuperAdmin { + organizationID = actor.OrganizationID + } + members, err := handler.service.ListAccounts(r.Context(), actor, administration.AccountFilters{OrganizationID: organizationID, IncludeDisabled: true}) + if err != nil { + writeAdminError(w, err) + return + } + organizations, err := handler.service.ListOrganizations(r.Context(), actor) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusOK, map[string]any{"configured": true, "currentOrganizationId": nullableAdminString(organizationID), "organizations": adminOrganizationProjections(organizations), "members": adminAccountProjections(members), "canManageOrganizations": actor.Role == administration.RoleSuperAdmin, "canAssignOrganizationAdmin": actor.Role == administration.RoleSuperAdmin, "canCreateSuperAdmin": actor.Role == administration.RoleSuperAdmin}) + case http.MethodPost: + body, ok := readAdminBody(w, r) + if !ok { + return + } + role := administration.Role(adminOptionalString(body["role"])) + if role == "" { + role = administration.RoleUser + } + account, err := handler.service.CreateAccount(r.Context(), actor, administration.CreateAccountInput{Phone: adminRequiredString(body["phone"]), DisplayName: adminRequiredString(body["displayName"]), Password: adminRequiredString(body["password"]), Role: role, OrganizationID: adminOptionalString(body["organizationId"]), LegacySubject: adminOptionalString(body["legacySubject"])}) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusCreated, map[string]any{"ok": true, "user": adminAccountProjection(administration.ProjectAccount(account))}) + case http.MethodPatch: + body, ok := readAdminBody(w, r) + if !ok { + return + } + patch := administration.UpdateAccountInput{ClearLoginLock: body["clearLoginLock"] == true} + if value, exists := body["displayName"]; exists { + v := adminOptionalString(value) + patch.DisplayName = &v + } + if value, exists := body["role"]; exists { + v := administration.Role(adminOptionalString(value)) + patch.Role = &v + } + if value, exists := body["organizationId"]; exists { + v := adminOptionalString(value) + patch.OrganizationID = &v + } + if value, exists := body["status"]; exists { + v := administration.Status(adminOptionalString(value)) + patch.Status = &v + } + account, err := handler.service.UpdateAccount(r.Context(), actor, adminRequiredString(body["userId"]), patch) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusOK, map[string]any{"ok": true, "user": adminAccountProjection(administration.ProjectAccount(account))}) + case http.MethodPut: + body, ok := readAdminBody(w, r) + if !ok { + return + } + account, err := handler.service.SetAccountStatus(r.Context(), actor, adminRequiredString(body["userId"]), administration.Status(adminRequiredString(body["status"]))) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusOK, map[string]any{"ok": true, "user": adminAccountProjection(administration.ProjectAccount(account))}) + case http.MethodDelete: + body, ok := readAdminBody(w, r) + if !ok { + return + } + archive, err := handler.service.DeleteAccount(r.Context(), actor, adminRequiredString(body["userId"])) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusOK, map[string]any{"ok": true, "archivedOwnerId": archive}) + default: + adminMethodNotAllowed(w, "GET, POST, PATCH, PUT, DELETE") + } + _ = session +} + +func (handler *adminHandler) servePasswordReset(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + adminMethodNotAllowed(w, http.MethodPost) + return + } + _, actor, ok := handler.authorize(w, r) + if !ok { + return + } + body, ok := readAdminBody(w, r) + if !ok { + return + } + account, err := handler.service.ResetPassword(r.Context(), actor, adminRequiredString(body["userId"]), adminRequiredString(body["newPassword"])) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusOK, map[string]any{"ok": true, "userId": account.ID}) +} + +func (handler *adminHandler) serveOrganizations(w http.ResponseWriter, r *http.Request) { + _, actor, ok := handler.authorize(w, r) + if !ok { + return + } + switch r.Method { + case http.MethodGet: + organizations, err := handler.service.ListOrganizations(r.Context(), actor) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusOK, map[string]any{"organizations": adminOrganizationProjections(organizations)}) + case http.MethodPost: + body, ok := readAdminBody(w, r) + if !ok { + return + } + organization, err := handler.service.CreateOrganization(r.Context(), actor, adminRequiredString(body["name"])) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusCreated, map[string]any{"ok": true, "organization": adminOrganizationProjection(administration.ProjectOrganization(organization))}) + case http.MethodPatch: + body, ok := readAdminBody(w, r) + if !ok { + return + } + patch := administration.UpdateOrganizationInput{} + if value, exists := body["name"]; exists { + v := adminOptionalString(value) + patch.Name = &v + } + if value, exists := body["status"]; exists { + v := administration.Status(adminOptionalString(value)) + patch.Status = &v + } + organization, err := handler.service.UpdateOrganization(r.Context(), actor, adminRequiredString(body["organizationId"]), patch) + if err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusOK, map[string]any{"ok": true, "organization": adminOrganizationProjection(administration.ProjectOrganization(organization))}) + case http.MethodDelete: + body, ok := readAdminBody(w, r) + if !ok { + return + } + if err := handler.service.DeleteOrganization(r.Context(), actor, adminRequiredString(body["organizationId"])); err != nil { + writeAdminError(w, err) + return + } + writeAdminJSON(w, http.StatusOK, map[string]any{"ok": true}) + default: + adminMethodNotAllowed(w, "GET, POST, PATCH, DELETE") + } +} + +func readAdminBody(w http.ResponseWriter, r *http.Request) (map[string]any, bool) { + var body map[string]any + if json.NewDecoder(r.Body).Decode(&body) != nil { + writeAdminJSON(w, http.StatusBadRequest, map[string]any{"error": "请求内容格式不正确。"}) + return nil, false + } + return body, true +} +func adminOptionalString(value any) string { + text, ok := value.(string) + if !ok { + return "" + } + return strings.TrimSpace(text) +} +func adminRequiredString(value any) string { return adminOptionalString(value) } +func nullableAdminString(value string) any { + if value == "" { + return nil + } + return value +} +func adminMethodNotAllowed(w http.ResponseWriter, allow string) { + w.Header().Set("Allow", allow) + w.WriteHeader(http.StatusMethodNotAllowed) +} +func writeAdminError(w http.ResponseWriter, err error) { + status, message := http.StatusInternalServerError, "服务器内部错误。" + var authErr *PlatformAuthError + if errors.As(err, &authErr) { + status, message = authErr.Status, authErr.Message + } else if administration.StatusCode(err) != http.StatusInternalServerError { + status, message = administration.StatusCode(err), err.Error() + } + writeAdminJSON(w, status, map[string]any{"error": message}) +} +func writeAdminJSON(w http.ResponseWriter, status int, body any) { writeJSON(w, status, body) } + +func adminAccountProjection(a administration.AccountProjection) map[string]any { + return map[string]any{"id": a.ID, "phone": a.Phone, "displayName": a.DisplayName, "role": a.Role, "organizationId": nullableAdminString(a.OrganizationID), "status": a.Status, "createdAt": a.CreatedAt, "lastLoginAt": a.LastLoginAt, "lockedUntil": a.LockedUntil} +} +func adminAccountProjections(values []administration.AccountProjection) []map[string]any { + out := make([]map[string]any, len(values)) + for i, value := range values { + out[i] = adminAccountProjection(value) + } + return out +} +func adminOrganizationProjection(o administration.OrganizationProjection) map[string]any { + return map[string]any{"id": o.ID, "name": o.Name, "status": o.Status} +} +func adminOrganizationProjections(values []administration.OrganizationProjection) []map[string]any { + out := make([]map[string]any, len(values)) + for i, value := range values { + out[i] = adminOrganizationProjection(value) + } + return out +} diff --git a/backend/internal/httpapi/admin_test.go b/backend/internal/httpapi/admin_test.go new file mode 100644 index 0000000..aa59db4 --- /dev/null +++ b/backend/internal/httpapi/admin_test.go @@ -0,0 +1,75 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration" +) + +func TestAdminGroupsCompatibilityIsGone(t *testing.T) { + authorizer, _ := NewPlatformAuthorizer(AuthState{Required: true}, nil) + handler, _ := NewAdminHandler(authorizer, administration.NewService(&adminStoreStub{})) + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/admin/accounts/groups", nil)) + if w.Code != http.StatusGone || !strings.Contains(w.Body.String(), "不再使用外部部门接口") { + t.Fatalf("status=%d body=%q", w.Code, w.Body.String()) + } +} + +func TestAdminAccountsListUsesSecurityProjection(t *testing.T) { + store := &adminStoreStub{accounts: []administration.Account{{ID: "u", Phone: "13800138000", DisplayName: "User", Role: administration.RoleUser, OrganizationID: "org", Status: administration.StatusActive, PasswordHash: "must-not-leak", PasswordSalt: "must-not-leak", SessionVersion: 9}}} + authorizer, _ := NewPlatformAuthorizer(AuthState{}, nil) + handler, _ := NewAdminHandler(authorizer, administration.NewService(store)) + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/admin/accounts", nil)) + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "must-not-leak") || strings.Contains(w.Body.String(), "sessionVersion") { + t.Fatalf("status=%d body=%q", w.Code, w.Body.String()) + } + var body struct { + Members []map[string]any `json:"members"` + CanManage bool `json:"canManageOrganizations"` + } + if json.Unmarshal(w.Body.Bytes(), &body) != nil || len(body.Members) != 1 || !body.CanManage { + t.Fatalf("body=%q", w.Body.String()) + } +} + +type adminStoreStub struct { + accounts []administration.Account + organizations []administration.Organization +} + +func (s *adminStoreStub) ListAccounts(context.Context, administration.AccountFilters) ([]administration.Account, error) { + return s.accounts, nil +} +func (s *adminStoreStub) GetAccount(context.Context, string) (administration.Account, bool, error) { + return administration.Account{}, false, nil +} +func (s *adminStoreStub) CreateAccount(_ context.Context, a administration.Account) (administration.Account, error) { + return a, nil +} +func (s *adminStoreStub) UpdateAccount(_ context.Context, a administration.Account) (administration.Account, error) { + return a, nil +} +func (s *adminStoreStub) DeleteAccount(context.Context, string, string) error { return nil } +func (s *adminStoreStub) ListOrganizations(context.Context, bool) ([]administration.Organization, error) { + return s.organizations, nil +} +func (s *adminStoreStub) GetOrganization(context.Context, string) (administration.Organization, bool, error) { + return administration.Organization{}, false, nil +} +func (s *adminStoreStub) CreateOrganization(_ context.Context, o administration.Organization) (administration.Organization, error) { + return o, nil +} +func (s *adminStoreStub) UpdateOrganization(_ context.Context, o administration.Organization) (administration.Organization, error) { + return o, nil +} +func (s *adminStoreStub) DeleteOrganization(context.Context, string) error { return nil } +func (s *adminStoreStub) CountOrganizationMembers(context.Context, string) (int, error) { + return 0, nil +} diff --git a/backend/internal/httpapi/assets.go b/backend/internal/httpapi/assets.go new file mode 100644 index 0000000..de93aa7 --- /dev/null +++ b/backend/internal/httpapi/assets.go @@ -0,0 +1,472 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/publicapi" +) + +const ( + defaultAssetJSONBytes int64 = 1 << 20 + defaultAssetUploadBytes int64 = 20 << 20 +) + +type PublicAssetAuthenticator interface { + Authenticate(*http.Request) (publicapi.PublicClient, string, error) +} + +type AssetsConfig struct { + MaxJSONBytes int64 + MaxUploadBytes int64 +} + +type assetsHandler struct { + service *assets.Service + platform *PlatformAuthorizer + public PublicAssetAuthenticator + config AssetsConfig +} + +func NewAssetsHandler(service *assets.Service, platform *PlatformAuthorizer, public PublicAssetAuthenticator, config AssetsConfig) (http.Handler, error) { + if service == nil || platform == nil || public == nil { + return nil, errors.New("assets HTTP dependencies are not configured") + } + if config.MaxJSONBytes <= 0 { + config.MaxJSONBytes = defaultAssetJSONBytes + } + if config.MaxUploadBytes <= 0 { + config.MaxUploadBytes = defaultAssetUploadBytes + } + return &assetsHandler{service: service, platform: platform, public: public, config: config}, nil +} + +func (h *assetsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + route, values := matchAssetRoute(r.URL.Path) + if route == "" { + http.NotFound(w, r) + return + } + allow := assetRouteAllow(route) + if r.Method == http.MethodOptions { + w.Header().Set("Allow", allow) + w.WriteHeader(http.StatusNoContent) + return + } + method := r.Method + if method == http.MethodHead && strings.Contains(allow, http.MethodHead) { + method = http.MethodGet + w = headResponseWriter{ResponseWriter: w} + } + if !methodAllowed(allow, method) { + w.Header().Set("Allow", allow) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + switch route { + case "platform-collection": + h.platformCollection(w, r, method) + case "platform-upload": + h.upload(w, r, false) + case "platform-item": + h.delete(w, r, values[0]) + case "platform-download": + h.download(w, r, false, values[0]) + case "public-collection": + h.publicCollection(w, r, method) + case "public-item": + h.publicGet(w, r, values[0]) + case "public-download": + h.download(w, r, true, values[0]) + case "served-file": + h.serveStored(w, r, values[0]) + } +} + +func (h *assetsHandler) platformCollection(w http.ResponseWriter, r *http.Request, method string) { + scope, ok := h.platformScope(w, r) + if !ok { + return + } + if method == http.MethodGet { + values, err := h.service.List(r.Context(), scope) + if err != nil { + writeAssetError(w, err, false, "") + return + } + if values == nil { + values = []assets.Asset{} + } + writeJSON(w, http.StatusOK, map[string]any{"assets": values}) + return + } + var input createAssetInput + if !decodeAssetJSON(w, r, h.config.MaxJSONBytes, &input) { + return + } + command := assets.CreateExternalCommand{URL: input.URL, Name: input.Name, Kind: input.Kind, Source: input.Source, Tags: input.Tags} + created, err := h.service.CreateExternal(r.Context(), scope, command) + if err != nil { + writeAssetError(w, err, false, "") + return + } + writeJSON(w, http.StatusCreated, map[string]any{"asset": created}) +} + +func (h *assetsHandler) publicCollection(w http.ResponseWriter, r *http.Request, method string) { + client, scope, ok := h.publicScope(w, r) + if !ok { + return + } + if method == http.MethodGet { + values, err := h.service.List(r.Context(), scope) + if err != nil { + writeAssetError(w, err, true, "") + return + } + if values == nil { + values = []assets.Asset{} + } + writeJSON(w, http.StatusOK, map[string]any{"assets": values}) + return + } + if strings.Contains(strings.ToLower(r.Header.Get("Content-Type")), "multipart/form-data") { + h.uploadScope(w, r, scope, true) + return + } + _ = client + var input createAssetInput + if !decodeAssetJSON(w, r, h.config.MaxJSONBytes, &input) { + return + } + command := assets.CreateExternalCommand{URL: input.URL, Name: input.Name, Kind: input.Kind, Tags: input.Tags} + created, err := h.service.CreateExternal(r.Context(), scope, command) + if err != nil { + writeAssetError(w, err, true, "") + return + } + writeJSON(w, http.StatusCreated, map[string]any{"asset": created}) +} + +type createAssetInput struct { + URL string `json:"url"` + Name string `json:"name"` + Kind assets.Kind `json:"kind"` + Source assets.Source `json:"source"` + Tags []string `json:"tags"` +} + +func (h *assetsHandler) upload(w http.ResponseWriter, r *http.Request, public bool) { + scope, ok := h.platformScope(w, r) + if !ok { + return + } + h.uploadScope(w, r, scope, public) +} + +func (h *assetsHandler) uploadScope(w http.ResponseWriter, r *http.Request, scope assets.Scope, public bool) { + r.Body = http.MaxBytesReader(w, r.Body, h.config.MaxUploadBytes) + reader, err := r.MultipartReader() + if err != nil { + writeAssetError(w, err, public, "") + return + } + type pendingUpload struct { + data []byte + fileName, contentType string + } + pending := make([]pendingUpload, 0) + for { + part, nextErr := reader.NextPart() + if errors.Is(nextErr, io.EOF) { + break + } + if nextErr != nil { + writeMultipartError(w, nextErr, public) + return + } + if part.FormName() != "files" || part.FileName() == "" { + _ = part.Close() + continue + } + data, readErr := io.ReadAll(part) + _ = part.Close() + if readErr != nil { + writeMultipartError(w, readErr, public) + return + } + contentType := part.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + pending = append(pending, pendingUpload{data: data, fileName: part.FileName(), contentType: contentType}) + } + if len(pending) == 0 { + writeAssetJSONError(w, http.StatusBadRequest, "No files uploaded.") + return + } + created := make([]assets.Asset, 0, len(pending)) + for _, file := range pending { + a, createErr := h.service.Upload(r.Context(), scope, assets.UploadCommand{Bytes: file.data, FileName: file.fileName, ContentType: file.contentType, Origin: requestOrigin(r)}) + if createErr != nil { + writeAssetError(w, createErr, public, "") + return + } + created = append(created, a) + } + writeJSON(w, http.StatusCreated, map[string]any{"assets": created}) +} + +func (h *assetsHandler) delete(w http.ResponseWriter, r *http.Request, id string) { + scope, ok := h.platformScope(w, r) + if !ok { + return + } + _, err := h.service.Delete(r.Context(), scope, id) + if err != nil { + writeAssetError(w, err, false, "资产不存在") + return + } + writeJSON(w, http.StatusOK, map[string]any{"ok": true, "deletedAssetId": id}) +} + +func (h *assetsHandler) publicGet(w http.ResponseWriter, r *http.Request, id string) { + _, scope, ok := h.publicScope(w, r) + if !ok { + return + } + a, err := h.service.Get(r.Context(), scope, id) + if err != nil { + writeAssetError(w, err, true, "Asset not found.") + return + } + writeJSON(w, http.StatusOK, map[string]any{"asset": a}) +} + +func (h *assetsHandler) download(w http.ResponseWriter, r *http.Request, public bool, id string) { + var scope assets.Scope + var ok bool + if public { + _, scope, ok = h.publicScope(w, r) + } else { + scope, ok = h.platformScope(w, r) + } + if !ok { + return + } + a, err := h.service.Get(r.Context(), scope, id) + if err != nil { + if public { + writeAssetError(w, err, true, "Asset not found.") + } else { + writeAssetError(w, err, false, "资产不存在") + } + return + } + blob, err := h.service.Download(r.Context(), scope, id) + if err != nil { + if public { + writeAssetError(w, err, true, "Asset file is not downloadable.") + } else { + writeAssetError(w, err, false, "资产文件不可下载") + } + return + } + defer blob.Body.Close() + writeBlob(w, blob, "private, no-store", contentDisposition(a.Name)) +} + +func (h *assetsHandler) serveStored(w http.ResponseWriter, r *http.Request, key string) { + scope, ok := h.platformScope(w, r) + if !ok { + return + } + blob, err := h.service.DownloadPath(r.Context(), scope, key) + if err != nil { + http.Error(w, "Not found", http.StatusNotFound) + return + } + defer blob.Body.Close() + writeBlob(w, blob, "public, max-age=31536000, immutable", "") +} + +func (h *assetsHandler) platformScope(w http.ResponseWriter, r *http.Request) (assets.Scope, bool) { + session, err := h.platform.Authorize(r, PlatformApp) + if err != nil { + writeAssetError(w, err, false, "") + return assets.Scope{}, false + } + return assets.PlatformScope(session.User.ID), true +} +func (h *assetsHandler) publicScope(w http.ResponseWriter, r *http.Request) (publicapi.PublicClient, assets.Scope, bool) { + client, _, err := h.public.Authenticate(r) + if err != nil { + writeAssetError(w, err, true, "") + return publicapi.PublicClient{}, assets.Scope{}, false + } + return client, assets.PublicScope(client.ID), true +} + +func decodeAssetJSON(w http.ResponseWriter, r *http.Request, limit int64, target any) bool { + r.Body = http.MaxBytesReader(w, r.Body, limit) + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(target); err != nil { + if isTooLarge(err) { + writeAssetJSONError(w, http.StatusRequestEntityTooLarge, "Request body is too large.") + } else { + writeAssetJSONError(w, http.StatusBadRequest, "Invalid request body.") + } + return false + } + return true +} +func writeMultipartError(w http.ResponseWriter, err error, public bool) { + if isTooLarge(err) { + writeAssetJSONError(w, http.StatusRequestEntityTooLarge, "Request body is too large.") + return + } + writeAssetError(w, err, public, "") +} +func isTooLarge(err error) bool { var max *http.MaxBytesError; return errors.As(err, &max) } +func writeAssetError(w http.ResponseWriter, err error, public bool, notFound string) { + if errors.Is(err, assets.ErrNotFound) || errors.Is(err, assets.ErrBlobNotFound) { + if notFound == "" { + if public { + notFound = "Asset not found." + } else { + notFound = "资产不存在" + } + } + writeAssetJSONError(w, http.StatusNotFound, notFound) + return + } + var platformErr *PlatformAuthError + if errors.As(err, &platformErr) { + writeAssetJSONError(w, platformErr.Status, platformErr.Message) + return + } + var publicErr *publicapi.AuthError + if errors.As(err, &publicErr) { + writeAssetJSONError(w, publicErr.Status, publicErr.Message) + return + } + if err != nil && (err.Error() == "url is required" || strings.Contains(err.Error(), "multipart")) { + writeAssetJSONError(w, http.StatusBadRequest, err.Error()) + return + } + writeAssetJSONError(w, http.StatusInternalServerError, "Internal server error.") +} +func writeAssetJSONError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]string{"error": message}) +} +func writeBlob(w http.ResponseWriter, blob assets.Blob, cache, disposition string) { + if blob.ContentType == "" { + blob.ContentType = "application/octet-stream" + } + w.Header().Set("Content-Type", blob.ContentType) + if blob.Size >= 0 { + w.Header().Set("Content-Length", fmt.Sprint(blob.Size)) + } + w.Header().Set("Cache-Control", cache) + if disposition != "" { + w.Header().Set("Content-Disposition", disposition) + } + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, blob.Body) +} +func contentDisposition(name string) string { + clean := strings.TrimSpace(strings.NewReplacer("\r", "_", "\n", "_", "/", "_", "\\", "_").Replace(name)) + if clean == "" { + clean = "download" + } + var ascii strings.Builder + for _, r := range clean { + if r >= 0x20 && r <= 0x7e && r != '"' { + ascii.WriteRune(r) + } else { + ascii.WriteByte('_') + } + } + return `attachment; filename="` + ascii.String() + `"; filename*=UTF-8''` + url.PathEscape(clean) +} +func requestOrigin(r *http.Request) string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + if forwarded := r.Header.Get("X-Forwarded-Proto"); forwarded != "" { + scheme = strings.TrimSpace(strings.Split(forwarded, ",")[0]) + } + return scheme + "://" + r.Host +} +func methodAllowed(allow, method string) bool { + for _, v := range strings.Split(allow, ",") { + if strings.TrimSpace(v) == method { + return true + } + } + return false +} + +type headResponseWriter struct{ http.ResponseWriter } + +func (headResponseWriter) Write(p []byte) (int, error) { return len(p), nil } + +func matchAssetRoute(value string) (string, []string) { + if value == "/api/assets" { + return "platform-collection", nil + } + if value == "/api/assets/upload" { + return "platform-upload", nil + } + if value == "/api/v1/assets" { + return "public-collection", nil + } + if strings.HasPrefix(value, "/api/assets/") { + rest := strings.TrimPrefix(value, "/api/assets/") + if rest != "" && !strings.Contains(rest, "/") { + return "platform-item", []string{rest} + } + if strings.HasSuffix(rest, "/download") && strings.Count(rest, "/") == 1 { + return "platform-download", []string{strings.TrimSuffix(rest, "/download")} + } + } + if strings.HasPrefix(value, "/api/v1/assets/") { + rest := strings.TrimPrefix(value, "/api/v1/assets/") + if rest != "" && !strings.Contains(rest, "/") { + return "public-item", []string{rest} + } + if strings.HasSuffix(rest, "/download") && strings.Count(rest, "/") == 1 { + return "public-download", []string{strings.TrimSuffix(rest, "/download")} + } + } + for _, prefix := range []string{"/uploads/", "/generated-results/"} { + if strings.HasPrefix(value, prefix) { + rest := strings.TrimPrefix(value, "/") + if rest != "" && path.Clean(rest) == rest && !strings.Contains(rest, "\\") { + return "served-file", []string{rest} + } + } + } + return "", nil +} +func assetRouteAllow(route string) string { + switch route { + case "platform-collection", "public-collection": + return "GET, HEAD, POST, OPTIONS" + case "platform-upload": + return "POST, OPTIONS" + case "platform-item": + return "DELETE, OPTIONS" + default: + return "GET, HEAD, OPTIONS" + } +} diff --git a/backend/internal/httpapi/assets_test.go b/backend/internal/httpapi/assets_test.go new file mode 100644 index 0000000..425c360 --- /dev/null +++ b/backend/internal/httpapi/assets_test.go @@ -0,0 +1,191 @@ +package httpapi_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/httpapi" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/publicapi" +) + +type assetCatalog struct{ values []assets.Asset } + +func (c *assetCatalog) ListOwner(_ context.Context, owner string) ([]assets.Asset, error) { + var v []assets.Asset + for _, a := range c.values { + if a.OwnerID == owner { + v = append(v, a) + } + } + return v, nil +} +func (c *assetCatalog) GetOwner(_ context.Context, owner, id string) (assets.Asset, bool, error) { + for _, a := range c.values { + if a.OwnerID == owner && a.ID == id { + return a, true, nil + } + } + return assets.Asset{}, false, nil +} +func (c *assetCatalog) GetOwnerByStoragePath(_ context.Context, owner, key string) (assets.Asset, bool, error) { + for _, a := range c.values { + if a.OwnerID == owner && a.StoragePath == key { + return a, true, nil + } + } + return assets.Asset{}, false, nil +} +func (c *assetCatalog) ListPublic(ctx context.Context, owner, client string, _ int) ([]assets.Asset, error) { + all, _ := c.ListOwner(ctx, owner) + var v []assets.Asset + for _, a := range all { + for _, tag := range a.Tags { + if tag == assets.ClientTag(client) { + v = append(v, a) + } + } + } + return v, nil +} +func (c *assetCatalog) GetPublic(ctx context.Context, owner, client, id string, _ int) (assets.Asset, bool, error) { + v, _ := c.ListPublic(ctx, owner, client, 0) + for _, a := range v { + if a.ID == id { + return a, true, nil + } + } + return assets.Asset{}, false, nil +} +func (c *assetCatalog) Create(_ context.Context, a assets.Asset) (assets.Asset, error) { + c.values = append(c.values, a) + return a, nil +} +func (c *assetCatalog) DeleteOwner(_ context.Context, owner, id string) (assets.Asset, bool, error) { + for i, a := range c.values { + if a.OwnerID == owner && a.ID == id { + c.values = append(c.values[:i], c.values[i+1:]...) + return a, true, nil + } + } + return assets.Asset{}, false, nil +} + +type assetBlobs struct{ values map[string][]byte } + +func (b *assetBlobs) Put(_ context.Context, key string, r io.Reader, _ int64, _ string) (assets.StoredObject, error) { + p, _ := io.ReadAll(r) + b.values[key] = p + return assets.StoredObject{Key: key, URL: "https://cdn.test/" + key}, nil +} +func (b *assetBlobs) Read(_ context.Context, key string) (assets.Blob, error) { + p, ok := b.values[key] + if !ok { + return assets.Blob{}, assets.ErrBlobNotFound + } + return assets.Blob{Body: io.NopCloser(bytes.NewReader(p)), ContentType: "image/png", Size: int64(len(p))}, nil +} +func (b *assetBlobs) Delete(_ context.Context, key string) error { delete(b.values, key); return nil } + +func TestAssetsPlatformAndPublicHTTP(t *testing.T) { + cat := &assetCatalog{} + blobs := &assetBlobs{values: map[string][]byte{}} + svc := assets.NewService(cat, blobs, nil, func() time.Time { return time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) }, func(prefix string) string { return prefix + "-1" }) + platform, _ := httpapi.NewPlatformAuthorizer(httpapi.AuthState{}, nil) + h, err := httpapi.NewAssetsHandler(svc, platform, publicapi.NewAuthenticator(publicapi.Config{APIKeys: "agent-a:secret"}), httpapi.AssetsConfig{}) + if err != nil { + t.Fatal(err) + } + post := request(t, h, http.MethodPost, "/api/assets", strings.NewReader(`{"url":"https://example.test/a.png"}`), map[string]string{"Content-Type": "application/json"}) + if post.Code != 201 || !strings.Contains(post.Body.String(), `"name":"外部图片"`) { + t.Fatalf("post=%d %s", post.Code, post.Body.String()) + } + list := request(t, h, http.MethodGet, "/api/assets", nil, nil) + if list.Code != 200 || !strings.Contains(list.Body.String(), `"assets"`) { + t.Fatalf("list=%d %s", list.Code, list.Body.String()) + } + pubUnauthorized := request(t, h, http.MethodGet, "/api/v1/assets", nil, nil) + if pubUnauthorized.Code != 401 { + t.Fatalf("public unauth=%d %s", pubUnauthorized.Code, pubUnauthorized.Body.String()) + } + pub := request(t, h, http.MethodPost, "/api/v1/assets", strings.NewReader(`{"url":"https://example.test/p.png"}`), map[string]string{"Authorization": "Bearer secret", "Content-Type": "application/json"}) + if pub.Code != 201 || !strings.Contains(pub.Body.String(), "api-client:agent-a") { + t.Fatalf("public post=%d %s", pub.Code, pub.Body.String()) + } +} + +func TestAssetsMultipartDownloadServingAndMethods(t *testing.T) { + cat := &assetCatalog{} + blobs := &assetBlobs{values: map[string][]byte{}} + svc := assets.NewService(cat, blobs, nil, time.Now, func(prefix string) string { return prefix + "-x" }) + platform, _ := httpapi.NewPlatformAuthorizer(httpapi.AuthState{}, nil) + h, _ := httpapi.NewAssetsHandler(svc, platform, publicapi.NewAuthenticator(publicapi.Config{APIKeys: "a:k"}), httpapi.AssetsConfig{MaxUploadBytes: 1024}) + var body bytes.Buffer + mw := multipart.NewWriter(&body) + part, _ := mw.CreateFormFile("files", "a.png") + _, _ = part.Write([]byte("png")) + _ = mw.Close() + upload := request(t, h, http.MethodPost, "/api/assets/upload", &body, map[string]string{"Content-Type": mw.FormDataContentType()}) + if upload.Code != 201 { + t.Fatalf("upload=%d %s", upload.Code, upload.Body.String()) + } + var payload struct { + Assets []assets.Asset `json:"assets"` + } + _ = json.Unmarshal(upload.Body.Bytes(), &payload) + id := payload.Assets[0].ID + dl := request(t, h, http.MethodGet, "/api/assets/"+id+"/download", nil, nil) + if dl.Code != 200 || dl.Body.String() != "png" || dl.Header().Get("Cache-Control") != "private, no-store" || !strings.Contains(dl.Header().Get("Content-Disposition"), "attachment") { + t.Fatalf("download=%d %#v %q", dl.Code, dl.Header(), dl.Body.String()) + } + served := request(t, h, http.MethodGet, "/uploads/"+strings.TrimPrefix(payload.Assets[0].StoragePath, "uploads/"), nil, nil) + if served.Code != 200 || served.Header().Get("Cache-Control") != "public, max-age=31536000, immutable" { + t.Fatalf("served=%d %#v", served.Code, served.Header()) + } + head := request(t, h, http.MethodHead, "/api/assets", nil, nil) + if head.Code != 200 || head.Body.Len() != 0 { + t.Fatalf("head=%d %q", head.Code, head.Body.String()) + } + options := request(t, h, http.MethodOptions, "/api/assets", nil, nil) + if options.Code != 204 || options.Header().Get("Allow") != "GET, HEAD, POST, OPTIONS" { + t.Fatalf("options=%d allow=%q", options.Code, options.Header().Get("Allow")) + } + bad := request(t, h, http.MethodPatch, "/api/assets", nil, nil) + if bad.Code != 405 || bad.Body.Len() != 0 { + t.Fatalf("bad=%d %q", bad.Code, bad.Body.String()) + } +} + +func TestAssetsLimitsAndInfrastructureErrorsDoNotLeak(t *testing.T) { + cat := &assetCatalog{} + svc := assets.NewService(cat, &assetBlobs{values: map[string][]byte{}}, nil, time.Now, nil) + platform, _ := httpapi.NewPlatformAuthorizer(httpapi.AuthState{}, nil) + h, _ := httpapi.NewAssetsHandler(svc, platform, publicapi.NewAuthenticator(publicapi.Config{APIKeys: "a:k"}), httpapi.AssetsConfig{MaxJSONBytes: 8, MaxUploadBytes: 8}) + r := request(t, h, http.MethodPost, "/api/assets", strings.NewReader(`{"url":"https://secret.example.test"}`), map[string]string{"Content-Type": "application/json"}) + if r.Code != 413 || strings.Contains(r.Body.String(), "secret") { + t.Fatalf("limit=%d %s", r.Code, r.Body.String()) + } + missing := request(t, h, http.MethodGet, "/api/assets/missing/download", nil, nil) + if missing.Code != 404 || missing.Body.String() != "{\"error\":\"资产不存在\"}\n" { + t.Fatalf("missing=%d %q", missing.Code, missing.Body.String()) + } +} + +func request(t *testing.T, h http.Handler, method, path string, body io.Reader, headers map[string]string) *httptest.ResponseRecorder { + t.Helper() + r := httptest.NewRequest(method, path, body) + for k, v := range headers { + r.Header.Set(k, v) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w +} diff --git a/backend/internal/httpapi/auth_compat.go b/backend/internal/httpapi/auth_compat.go new file mode 100644 index 0000000..dbb1d59 --- /dev/null +++ b/backend/internal/httpapi/auth_compat.go @@ -0,0 +1,41 @@ +package httpapi + +import ( + "net/http" + "net/url" +) + +// NewAuthCompatibilityHandler preserves the legacy external-login endpoints +// after platform password login became the only supported authentication flow. +func NewAuthCompatibilityHandler() http.Handler { return authCompatibilityHandler{} } + +type authCompatibilityHandler struct{} + +func (authCompatibilityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/auth/login" && r.URL.Path != "/api/auth/callback" && r.URL.Path != "/api/auth/captcha" { + http.NotFound(w, r) + return + } + if r.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if r.URL.Path == "/api/auth/captcha" { + writeJSON(w, http.StatusOK, map[string]any{"enabled": false, "message": "平台账号登录不使用外部验证码。"}) + return + } + location := "/auth/login" + if r.URL.Path == "/api/auth/callback" { + location += "?error=callback_failed" + } + if base, err := url.Parse(absoluteRequestURL(r)); err == nil && base.IsAbs() { + base.Path, base.RawPath, base.RawQuery, base.Fragment = "/auth/login", "", "", "" + if r.URL.Path == "/api/auth/callback" { + base.RawQuery = "error=callback_failed" + } + location = base.String() + } + w.Header().Set("Location", location) + w.WriteHeader(http.StatusTemporaryRedirect) +} diff --git a/backend/internal/httpapi/auth_compat_test.go b/backend/internal/httpapi/auth_compat_test.go new file mode 100644 index 0000000..c66a5dc --- /dev/null +++ b/backend/internal/httpapi/auth_compat_test.go @@ -0,0 +1,39 @@ +package httpapi + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestAuthCompatibilityEndpoints(t *testing.T) { + handler := NewAuthCompatibilityHandler() + cases := []struct { + path string + status int + location string + body string + }{ + {"/api/auth/login", http.StatusTemporaryRedirect, "https://example.test/auth/login", ""}, + {"/api/auth/callback", http.StatusTemporaryRedirect, "https://example.test/auth/login?error=callback_failed", ""}, + {"/api/auth/captcha", http.StatusOK, "", `{"enabled":false,"message":"平台账号登录不使用外部验证码。"}` + "\n"}, + } + for _, tc := range cases { + t.Run(tc.path, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "https://example.test"+tc.path, nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Code != tc.status || w.Header().Get("Location") != tc.location || w.Body.String() != tc.body { + t.Fatalf("status=%d location=%q body=%q", w.Code, w.Header().Get("Location"), w.Body.String()) + } + }) + } +} + +func TestAuthCompatibilityRejectsUnsupportedMethods(t *testing.T) { + w := httptest.NewRecorder() + NewAuthCompatibilityHandler().ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/auth/captcha", nil)) + if w.Code != http.StatusMethodNotAllowed || w.Header().Get("Allow") != http.MethodGet { + t.Fatalf("status=%d allow=%q", w.Code, w.Header().Get("Allow")) + } +} diff --git a/backend/internal/httpapi/auth_password_change.go b/backend/internal/httpapi/auth_password_change.go new file mode 100644 index 0000000..4334e84 --- /dev/null +++ b/backend/internal/httpapi/auth_password_change.go @@ -0,0 +1,130 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" +) + +type PasswordChangeSessionIssuer interface { + Change(context.Context, identity.PasswordChangeCommand) (identity.Session, error) +} + +type PasswordChangeConfig struct { + SessionSecret, CookieSecure, PublicBaseURL string +} + +type authPasswordChangeHandler struct { + config PasswordChangeConfig + authorizer *PlatformAuthorizer + changer PasswordChangeSessionIssuer +} + +func NewAuthPasswordChangeHandler(config PasswordChangeConfig, authorizer *PlatformAuthorizer, changer PasswordChangeSessionIssuer) (http.Handler, error) { + if authorizer == nil || changer == nil || strings.TrimSpace(config.SessionSecret) == "" { + return nil, fmt.Errorf("auth/password/change: authorizer, password changer, and session secret are required") + } + return &authPasswordChangeHandler{config: config, authorizer: authorizer, changer: changer}, nil +} + +func (handler *authPasswordChangeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/auth/password/change" { + http.NotFound(w, r) + return + } + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + session, err := handler.authorizer.Authorize(r, PlatformApp) + if err != nil { + writePasswordChangeError(w, err) + return + } + var body struct{ CurrentPassword, NewPassword, ConfirmPassword any } + if json.NewDecoder(r.Body).Decode(&body) != nil { + writePasswordChangeError(w, &identity.PasswordChangeError{Reason: identity.PasswordChangeInvalidInput}) + return + } + current := passwordChangeString(body.CurrentPassword) + next := passwordChangeString(body.NewPassword) + confirm := passwordChangeString(body.ConfirmPassword) + if current == "" || next == "" || confirm == "" { + writePasswordChangeError(w, &identity.PasswordChangeError{Reason: identity.PasswordChangeInvalidInput}) + return + } + if next != confirm { + writePasswordJSON(w, http.StatusBadRequest, map[string]any{"error": "两次输入的新密码不一致。"}) + return + } + nextSession, err := handler.changer.Change(r.Context(), identity.PasswordChangeCommand{AccountID: session.User.ID, CurrentPassword: current, NewPassword: next}) + if err != nil { + writePasswordChangeError(w, err) + return + } + raw, err := json.Marshal(nextSession) + if err != nil { + writePasswordChangeError(w, err) + return + } + signed, err := identity.Sign(raw, handler.config.SessionSecret) + if err != nil { + writePasswordChangeError(w, err) + return + } + secure := identity.ResolveSecureCookie(handler.config.CookieSecure, handler.config.PublicBaseURL, absoluteRequestURL(r)) + writes, err := identity.SetSessionCookies(signed, time.Unix(nextSession.ExpiresAt, 0).UTC(), secure) + if err != nil { + writePasswordChangeError(w, err) + return + } + payload, err := json.Marshal(map[string]any{"ok": true, "user": passwordPublicUser(nextSession.User)}) + if err != nil { + writePasswordChangeError(w, err) + return + } + for _, write := range writes { + http.SetCookie(w, transportCookie(write)) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(payload) +} + +func passwordChangeString(value any) string { + text, ok := value.(string) + if !ok { + return "" + } + return strings.TrimSpace(text) +} + +func writePasswordChangeError(w http.ResponseWriter, err error) { + status, message := http.StatusInternalServerError, "服务器内部错误。" + var authErr *PlatformAuthError + if errors.As(err, &authErr) { + status, message = authErr.Status, authErr.Message + } + var changeErr *identity.PasswordChangeError + if errors.As(err, &changeErr) { + status = http.StatusBadRequest + switch changeErr.Reason { + case identity.PasswordChangeNotFound: + status, message = http.StatusNotFound, "账号不存在或已停用。" + case identity.PasswordChangeCurrentIncorrect: + message = "当前密码不正确。" + case identity.PasswordChangeInvalidNewPassword: + message = "新密码至少需要 8 位。" + default: + message = "当前密码、新密码和确认密码不能为空。" + } + } + writePasswordJSON(w, status, map[string]any{"error": message}) +} diff --git a/backend/internal/httpapi/auth_password_change_test.go b/backend/internal/httpapi/auth_password_change_test.go new file mode 100644 index 0000000..7b35be0 --- /dev/null +++ b/backend/internal/httpapi/auth_password_change_test.go @@ -0,0 +1,71 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" +) + +type passwordChangerStub struct { + command identity.PasswordChangeCommand + session identity.Session + err error +} + +func (s *passwordChangerStub) Change(_ context.Context, command identity.PasswordChangeCommand) (identity.Session, error) { + s.command = command + return s.session, s.err +} + +func TestAuthPasswordChangeReplacesSignedSessionCookie(t *testing.T) { + secret := "password-change-secret-with-enough-entropy" + now := time.Now().UTC().Truncate(time.Second) + version := 4 + changer := &passwordChangerStub{session: identity.Session{Version: 1, AuthMode: identity.AuthModeUser, IssuedAt: now.Unix(), ExpiresAt: now.Add(time.Hour).Unix(), SessionVersion: &version, User: identity.User{ID: "u", Subject: "u", Phone: "13800138000", DisplayName: "User", ClientID: "platform", Role: "user", Status: "active", Authorities: []string{"ROLE_USER"}, Scope: []string{}}}} + resolver := &platformSessionResolverStub{outcome: "authenticated", session: identity.Session{AuthMode: identity.AuthModeUser, User: identity.User{ID: "u", Role: "user"}}} + authorizer, _ := NewPlatformAuthorizer(AuthState{Required: true, Configured: true}, resolver) + handler, err := NewAuthPasswordChangeHandler(PasswordChangeConfig{SessionSecret: secret}, authorizer, changer) + if err != nil { + t.Fatal(err) + } + r := httptest.NewRequest(http.MethodPost, "https://app.test/api/auth/password/change", strings.NewReader(`{"currentPassword":" current-password ","newPassword":"next-password","confirmPassword":"next-password"}`)) + r.AddCookie(&http.Cookie{Name: identity.SessionCookieName, Value: "existing"}) + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Code != http.StatusOK || changer.command.AccountID != "u" || changer.command.CurrentPassword != "current-password" { + t.Fatalf("status=%d command=%+v body=%q", w.Code, changer.command, w.Body.String()) + } + cookies := w.Result().Cookies() + if len(cookies) != identity.CookieMaxChunks { + t.Fatalf("cookies=%d", len(cookies)) + } + signed := cookies[0].Value + parsed, err := identity.Parse(signed, secret, now) + if err != nil || parsed.SessionVersion == nil || *parsed.SessionVersion != 4 { + t.Fatalf("parse=%+v err=%v", parsed, err) + } + var body map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["ok"] != true { + t.Fatalf("body=%q err=%v", w.Body.String(), err) + } +} + +func TestAuthPasswordChangeValidatesConfirmation(t *testing.T) { + resolver := &platformSessionResolverStub{outcome: "authenticated", session: identity.Session{AuthMode: identity.AuthModeUser, User: identity.User{ID: "u", Role: "user"}}} + authorizer, _ := NewPlatformAuthorizer(AuthState{Required: true, Configured: true}, resolver) + changer := &passwordChangerStub{} + handler, _ := NewAuthPasswordChangeHandler(PasswordChangeConfig{SessionSecret: "secret"}, authorizer, changer) + r := httptest.NewRequest(http.MethodPost, "/api/auth/password/change", strings.NewReader(`{"currentPassword":"old-password","newPassword":"new-password","confirmPassword":"different"}`)) + r.AddCookie(&http.Cookie{Name: identity.SessionCookieName, Value: "cookie"}) + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest || changer.command.AccountID != "" { + t.Fatalf("status=%d command=%+v", w.Code, changer.command) + } +} diff --git a/backend/internal/httpapi/billing.go b/backend/internal/httpapi/billing.go new file mode 100644 index 0000000..fc0ce2f --- /dev/null +++ b/backend/internal/httpapi/billing.go @@ -0,0 +1,337 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "math" + "net/http" + "strings" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/providers" +) + +type BillingHTTPService = billing.HTTPService +type BillingAccountStore = billing.AccountConfigStore + +type billingHandler struct { + authorizer *PlatformAuthorizer + service BillingHTTPService + accounts BillingAccountStore + builder jobs.ProviderJobBuilder +} + +func NewBillingHandler(authorizer *PlatformAuthorizer, service BillingHTTPService, accounts BillingAccountStore) http.Handler { + return &billingHandler{authorizer: authorizer, service: service, accounts: accounts} +} + +func NewBillingHandlerWithBuilder(authorizer *PlatformAuthorizer, service BillingHTTPService, accounts BillingAccountStore, builder jobs.ProviderJobBuilder) http.Handler { + return &billingHandler{authorizer: authorizer, service: service, accounts: accounts, builder: builder} +} + +func (h *billingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/billing": + h.billing(w, r) + case r.URL.Path == "/api/billing/quote": + h.quote(w, r) + case r.URL.Path == "/api/admin/billing": + h.adminOverview(w, r) + case r.URL.Path == "/api/admin/billing/account": + h.account(w, r) + case r.URL.Path == "/api/admin/billing/adjustments": + h.adjust(w, r) + case r.URL.Path == "/api/admin/billing/prices": + h.prices(w, r) + case strings.HasPrefix(r.URL.Path, "/api/admin/billing/prices/"): + h.price(w, r, strings.TrimPrefix(r.URL.Path, "/api/admin/billing/prices/")) + default: + http.NotFound(w, r) + } +} + +func (h *billingHandler) billing(w http.ResponseWriter, r *http.Request) { + if !allow(w, r, http.MethodGet) { + return + } + session, ok := h.authorize(w, r, PlatformApp) + if !ok { + return + } + if session.User.OrganizationID == "" { + writeAPIError(w, 422, "当前账号未绑定组织。") + return + } + if h.service == nil { + writeAPIError(w, 500, "服务器内部错误。") + return + } + overview, err := h.service.Overview(r.Context(), session.User.OrganizationID, session.User.ID) + if err != nil { + writeDomainError(w, err) + return + } + if h.accounts != nil { + config, err := h.accounts.Load(r.Context()) + if err != nil { + writeDomainError(w, err) + return + } + overview.BillingAccount = config + } + writeJSON(w, 200, struct { + Organization any `json:"organization"` + billing.Overview + }{Organization: map[string]string{"id": session.User.OrganizationID, "name": first(session.User.OrganizationName, session.User.OrganizationID)}, Overview: overview}) +} + +func (h *billingHandler) quote(w http.ResponseWriter, r *http.Request) { + if !allow(w, r, http.MethodPost) { + return + } + if h.service == nil { + writeAPIError(w, 500, "服务器内部错误。") + return + } + session, ok := h.authorize(w, r, PlatformApp) + if !ok { + return + } + var body map[string]any + decodeJSON(r, &body) + capability := billingString(body["capability"]) + if capability == "" { + if billingString(body["kind"]) == "video" { + capability = "video.generate" + } else { + capability = "image.generate" + } + } + provider, reqKey := billingString(body["provider"]), billingString(body["reqKey"]) + parameters := billing.Parameters{} + if h.builder.NewID != nil { + prepared, err := h.builder.Build(r.Context(), session.User.ID, "", capability, "", body) + if err != nil { + writeJobError(w, err, false) + return + } + provider, reqKey = prepared.Job.Provider, prepared.Job.ReqKey + var request providers.Request + if json.Unmarshal(prepared.Job.RequestPayload, &request) != nil { + writeAPIError(w, 500, "服务器内部错误。") + return + } + for key, value := range request.Settings { + parameters[key] = value + } + parameters["referenceImageCount"] = float64(len(request.InputURLs)) + } + command := billing.QuoteCommand{AccountID: session.User.ID, OrganizationID: session.User.OrganizationID, OrganizationName: session.User.OrganizationName, Role: session.User.Role, Provider: provider, Capability: capability, ReqKey: reqKey, Payload: body, Parameters: parameters} + if parameters, ok := body["parameters"].(map[string]any); ok { + for key, value := range parameters { + command.Parameters[key] = value + } + } + quote, err := h.service.Quote(r.Context(), command) + if err != nil { + writeDomainError(w, err) + return + } + writeJSON(w, 200, map[string]any{"quote": quote}) +} + +func (h *billingHandler) adminOverview(w http.ResponseWriter, r *http.Request) { + if !allow(w, r, http.MethodGet) { + return + } + if _, ok := h.authorize(w, r, PlatformSuperAdmin); !ok { + return + } + overview, err := h.service.AdminOverview(r.Context()) + if err != nil { + writeDomainError(w, err) + return + } + if h.accounts != nil { + overview.BillingAccount, err = h.accounts.Load(r.Context()) + if err != nil { + writeDomainError(w, err) + return + } + } + writeJSON(w, 200, overview) +} +func (h *billingHandler) account(w http.ResponseWriter, r *http.Request) { + if !allow(w, r, http.MethodPatch) { + return + } + if _, ok := h.authorize(w, r, PlatformSuperAdmin); !ok { + return + } + if h.accounts == nil { + writeAPIError(w, 500, "服务器内部错误。") + return + } + var body map[string]any + decodeJSON(r, &body) + config := billing.AccountConfig{AccountName: billingString(body["accountName"]), BankName: billingString(body["bankName"]), AccountNumber: billingString(body["accountNumber"]), Contact: billingString(body["contact"])} + if err := h.accounts.Save(r.Context(), config); err != nil { + writeDomainError(w, err) + return + } + config, err := h.accounts.Load(r.Context()) + if err != nil { + writeDomainError(w, err) + return + } + writeJSON(w, 200, map[string]any{"billingAccount": config}) +} +func (h *billingHandler) adjust(w http.ResponseWriter, r *http.Request) { + if !allow(w, r, http.MethodPost) { + return + } + session, ok := h.authorize(w, r, PlatformSuperAdmin) + if !ok { + return + } + var body map[string]any + decodeJSON(r, &body) + organizationID, note, direction := billingString(body["organizationId"]), billingString(body["note"]), billingString(body["direction"]) + amount := numberValue(body["amountFen"]) + if amount <= 0 { + amount = numberValue(body["amountYuan"]) * 100 + } + amountFen := int64(math.Round(amount)) + if organizationID == "" { + writeAPIError(w, 400, "组织不能为空。") + return + } + if amountFen <= 0 { + writeAPIError(w, 400, "请输入大于 0 的金额。") + return + } + if direction != "credit" && direction != "debit" { + writeAPIError(w, 400, "余额变动方向无效。") + return + } + if note == "" { + writeAPIError(w, 400, "备注不能为空。") + return + } + delta := amountFen + if direction == "debit" { + delta = -amountFen + } + result, err := h.service.Adjust(r.Context(), billing.AdjustmentCommand{OrganizationID: organizationID, OperatorID: session.User.ID, Direction: direction, Note: note, AmountFen: amountFen, DeltaFen: delta}) + if err != nil { + writeDomainError(w, err) + return + } + writeJSON(w, 200, result) +} +func (h *billingHandler) prices(w http.ResponseWriter, r *http.Request) { + if !allow(w, r, http.MethodGet) { + return + } + if _, ok := h.authorize(w, r, PlatformSuperAdmin); !ok { + return + } + rules, err := h.service.ListPrices(r.Context()) + if err != nil { + writeDomainError(w, err) + return + } + writeJSON(w, 200, map[string]any{"priceRules": rules}) +} +func (h *billingHandler) price(w http.ResponseWriter, r *http.Request, id string) { + if !allow(w, r, http.MethodPatch) { + return + } + if _, ok := h.authorize(w, r, PlatformSuperAdmin); !ok { + return + } + if id == "" || strings.Contains(id, "/") { + http.NotFound(w, r) + return + } + var body map[string]any + decodeJSON(r, &body) + for key := range body { + if key != "markupMultiplier" && key != "dimensionKey" && key != "tierValue" { + writeAPIError(w, 400, "平台标准价格与参数由系统维护,超管仅可调整上浮倍率。") + return + } + } + patch := billing.PricePatch{MarkupMultiplier: numberValue(body["markupMultiplier"]), DimensionKey: billingString(body["dimensionKey"]), TierValue: billingString(body["tierValue"])} + rule, err := h.service.GetPrice(r.Context(), id) + if err != nil { + writeDomainError(w, err) + return + } + if rule == nil { + writeAPIError(w, 404, "计费规则不存在") + return + } + if err := billing.ValidatePricePatch(rule, patch); err != nil { + writeAPIError(w, 400, err.Error()) + return + } + patch.MarkupMultiplier = math.Round(patch.MarkupMultiplier*10000) / 10000 + rule, err = h.service.UpdatePrice(r.Context(), id, patch) + if err != nil { + writeDomainError(w, err) + return + } + if rule == nil { + writeAPIError(w, 404, "计费规则不存在") + return + } + writeJSON(w, 200, map[string]any{"rule": rule}) +} + +func (h *billingHandler) authorize(w http.ResponseWriter, r *http.Request, requirement PlatformRequirement) (identity.Session, bool) { + session, err := h.authorizer.Authorize(r, requirement) + if err != nil { + writeAuthError(w, err) + return identity.Session{}, false + } + return session, true +} +func decodeJSON(r *http.Request, value any) { _ = json.NewDecoder(r.Body).Decode(value) } +func billingString(value any) string { text, _ := value.(string); return strings.TrimSpace(text) } +func numberValue(value any) float64 { number, _ := value.(float64); return number } +func first(value, fallback string) string { + if value != "" { + return value + } + return fallback +} +func allow(w http.ResponseWriter, r *http.Request, method string) bool { + if r.Method == method { + return true + } + w.Header().Set("Allow", method) + writeAPIError(w, 405, http.StatusText(405)) + return false +} +func writeAuthError(w http.ResponseWriter, err error) { + var auth *PlatformAuthError + if errors.As(err, &auth) { + writeAPIError(w, auth.Status, auth.Message) + return + } + writeAPIError(w, 500, "服务器内部错误。") +} +func writeDomainError(w http.ResponseWriter, err error) { + status := billing.HTTPStatus(err) + if status >= 500 { + writeAPIError(w, 500, "服务器内部错误。") + return + } + writeAPIError(w, status, err.Error()) +} +func writeAPIError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]string{"error": message}) +} diff --git a/backend/internal/httpapi/billing_test.go b/backend/internal/httpapi/billing_test.go new file mode 100644 index 0000000..5e05c96 --- /dev/null +++ b/backend/internal/httpapi/billing_test.go @@ -0,0 +1,177 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" +) + +func TestBillingMemberRoutesUseRefreshedSessionScope(t *testing.T) { + service := &billingHTTPServiceStub{} + h := billingTestHandler(t, identity.Session{AuthMode: identity.AuthModeUser, User: identity.User{ID: "db-user", ClientID: "platform", OrganizationID: "db-org", OrganizationName: "DB Org", Role: "user"}}, service, &billingAccountStoreStub{}) + + response := serveJSON(t, h, http.MethodGet, "/api/billing", nil) + if response.Code != 200 || service.overviewOrganization != "db-org" || service.overviewAccount != "db-user" { + t.Fatalf("status=%d scope=%s/%s body=%s", response.Code, service.overviewOrganization, service.overviewAccount, response.Body.String()) + } + response = serveJSON(t, h, http.MethodPost, "/api/billing/quote", map[string]any{"provider": "bailian", "capability": "image.generate", "ownerId": "attacker", "organizationId": "other"}) + if response.Code != 200 || service.quote.AccountID != "db-user" || service.quote.OrganizationID != "db-org" || service.quote.Role != "user" { + t.Fatalf("status=%d quote=%+v body=%s", response.Code, service.quote, response.Body.String()) + } +} + +func TestBillingQuotePreparesProviderAndModelFromServerOwnedEngineTargets(t *testing.T) { + service := &billingHTTPServiceStub{} + h := billingTestHandler(t, identity.Session{User: identity.User{ID: "db-user", ClientID: "platform", OrganizationID: "db-org", Role: "user"}}, service, nil) + h = NewBillingHandlerWithBuilder(h.(*billingHandler).authorizer, service, nil, jobs.ProviderJobBuilder{ + ImageEngine: "bailian", ImageProvider: "bailian", ImageModel: "wan-image", + ImageEngines: map[string]jobs.ProviderTarget{"evolink": {Provider: "evolink", Model: "gpt-image-2"}}, + VideoEngine: "seedance", VideoProvider: "seedance", VideoModel: "seedance-2", + NewID: func() string { return "quote-only" }, + }) + response := serveJSON(t, h, http.MethodPost, "/api/billing/quote", map[string]any{"engine": "evolink", "prompt": "hello", "quality": "high"}) + if response.Code != 200 || service.quote.Provider != "evolink" || service.quote.Capability != "image.generate" || service.quote.ReqKey != "gpt-image-2" || service.quote.Parameters["quality"] != "high" { + t.Fatalf("status=%d quote=%+v body=%s", response.Code, service.quote, response.Body.String()) + } +} + +func TestBillingRequiresOrganizationAndMapsDomainStatuses(t *testing.T) { + service := &billingHTTPServiceStub{} + h := billingTestHandler(t, identity.Session{AuthMode: identity.AuthModeUser, User: identity.User{ID: "user", ClientID: "platform", Role: "user"}}, service, nil) + if got := serveJSON(t, h, http.MethodGet, "/api/billing", nil); got.Code != 422 { + t.Fatalf("status=%d body=%s", got.Code, got.Body.String()) + } + + service.err = &billing.StatusError{Status: 402, Err: billing.ErrInsufficientBalance} + h = billingTestHandler(t, identity.Session{AuthMode: identity.AuthModeUser, User: identity.User{ID: "user", ClientID: "platform", OrganizationID: "org", Role: "user"}}, service, nil) + if got := serveJSON(t, h, http.MethodPost, "/api/billing/quote", map[string]any{}); got.Code != 402 { + t.Fatalf("status=%d body=%s", got.Code, got.Body.String()) + } + service.err = errors.New("database secret") + got := serveJSON(t, h, http.MethodGet, "/api/billing", nil) + if got.Code != 500 || bytes.Contains(got.Body.Bytes(), []byte("database secret")) { + t.Fatalf("status=%d body=%s", got.Code, got.Body.String()) + } +} + +func TestBillingAdminRoutesRequireSuperAdminAndValidateWrites(t *testing.T) { + service := &billingHTTPServiceStub{price: &billing.PriceRule{ID: "price-1", MarkupMultiplier: 1.2}} + account := &billingAccountStoreStub{} + orgAdmin := billingTestHandler(t, identity.Session{AuthMode: identity.AuthModeAdmin, User: identity.User{ID: "admin", ClientID: "platform", OrganizationID: "org", Role: "organization_admin"}}, service, account) + if got := serveJSON(t, orgAdmin, http.MethodGet, "/api/admin/billing", nil); got.Code != 403 { + t.Fatalf("organization admin status=%d", got.Code) + } + + super := billingTestHandler(t, identity.Session{AuthMode: identity.AuthModeAdmin, User: identity.User{ID: "root", ClientID: "platform", Role: "super_admin"}}, service, account) + if got := serveJSON(t, super, http.MethodPatch, "/api/admin/billing/account", map[string]any{"accountName": " Acme ", "bankName": " Bank ", "accountNumber": " 123 ", "contact": " Ops "}); got.Code != 200 || account.saved.AccountName != "Acme" { + t.Fatalf("account status=%d saved=%+v body=%s", got.Code, account.saved, got.Body.String()) + } + if got := serveJSON(t, super, http.MethodPost, "/api/admin/billing/adjustments", map[string]any{"organizationId": "org", "amountYuan": 1.235, "direction": "debit", "note": " correction "}); got.Code != 200 || service.adjustment.AmountFen != 124 || service.adjustment.DeltaFen != -124 || service.adjustment.OperatorID != "root" { + t.Fatalf("adjustment status=%d got=%+v body=%s", got.Code, service.adjustment, got.Body.String()) + } + if got := serveJSON(t, super, http.MethodPatch, "/api/admin/billing/prices/price-1", map[string]any{"standardUnitPriceFen": 1, "markupMultiplier": 2}); got.Code != 400 { + t.Fatalf("whitelist status=%d", got.Code) + } + if got := serveJSON(t, super, http.MethodPatch, "/api/admin/billing/prices/price-1", map[string]any{"markupMultiplier": .5}); got.Code != 400 { + t.Fatalf("multiplier status=%d", got.Code) + } + service.price.Dimensions = []billing.ParameterDimension{{Key: "quality", Tiers: []billing.ParameterTier{{Value: "high", Enabled: true}}}} + if got := serveJSON(t, super, http.MethodPatch, "/api/admin/billing/prices/price-1", map[string]any{"markupMultiplier": 2}); got.Code != 400 { + t.Fatalf("tier-required status=%d", got.Code) + } + if got := serveJSON(t, super, http.MethodPatch, "/api/admin/billing/prices/price-1", map[string]any{"markupMultiplier": 2, "dimensionKey": "quality", "tierValue": "missing"}); got.Code != 400 { + t.Fatalf("unknown tier status=%d", got.Code) + } + if got := serveJSON(t, super, http.MethodPatch, "/api/admin/billing/prices/price-1", map[string]any{"markupMultiplier": 2.123456, "dimensionKey": "quality", "tierValue": "high"}); got.Code != 200 || service.pricePatch.MarkupMultiplier != 2.1235 { + t.Fatalf("patch status=%d patch=%+v body=%s", got.Code, service.pricePatch, got.Body.String()) + } +} + +func TestBillingHandlerRejectsWrongMethods(t *testing.T) { + h := billingTestHandler(t, identity.Session{AuthMode: identity.AuthModeAdmin, User: identity.User{ID: "root", ClientID: "platform", Role: "super_admin"}}, &billingHTTPServiceStub{}, nil) + got := serveJSON(t, h, http.MethodDelete, "/api/billing", nil) + if got.Code != 405 || got.Header().Get("Allow") != http.MethodGet { + t.Fatalf("status=%d allow=%q", got.Code, got.Header().Get("Allow")) + } +} + +type billingHTTPServiceStub struct { + overviewOrganization, overviewAccount string + quote billing.QuoteCommand + adjustment billing.AdjustmentCommand + pricePatch billing.PricePatch + price *billing.PriceRule + err error +} + +func (s *billingHTTPServiceStub) Overview(_ context.Context, organizationID, accountID string) (billing.Overview, error) { + s.overviewOrganization, s.overviewAccount = organizationID, accountID + return billing.Overview{Wallet: billing.Wallet{OrganizationID: organizationID, Currency: billing.CurrencyCNY}}, s.err +} +func (s *billingHTTPServiceStub) Quote(_ context.Context, command billing.QuoteCommand) (*billing.Quote, error) { + s.quote = command + return &billing.Quote{Currency: billing.CurrencyCNY}, s.err +} +func (s *billingHTTPServiceStub) AdminOverview(context.Context) (billing.AdminOverview, error) { + return billing.AdminOverview{}, s.err +} +func (s *billingHTTPServiceStub) ListPrices(context.Context) ([]billing.PriceRule, error) { + return nil, s.err +} +func (s *billingHTTPServiceStub) GetPrice(_ context.Context, _ string) (*billing.PriceRule, error) { + return s.price, s.err +} +func (s *billingHTTPServiceStub) UpdatePrice(_ context.Context, _ string, patch billing.PricePatch) (*billing.PriceRule, error) { + s.pricePatch = patch + return s.price, s.err +} +func (s *billingHTTPServiceStub) Adjust(_ context.Context, command billing.AdjustmentCommand) (billing.AdjustmentResult, error) { + s.adjustment = command + return billing.AdjustmentResult{Wallet: billing.Wallet{OrganizationID: command.OrganizationID, Currency: billing.CurrencyCNY}}, s.err +} + +type billingAccountStoreStub struct{ saved billing.AccountConfig } + +func (s *billingAccountStoreStub) Load(context.Context) (billing.AccountConfig, error) { + return s.saved, nil +} +func (s *billingAccountStoreStub) Save(_ context.Context, value billing.AccountConfig) error { + s.saved = value + return nil +} + +func billingTestHandler(t *testing.T, session identity.Session, service BillingHTTPService, accounts BillingAccountStore) http.Handler { + t.Helper() + authorizer, err := NewPlatformAuthorizer(AuthState{Required: true, Configured: true}, &fixedSessionResolver{session: session}) + if err != nil { + t.Fatal(err) + } + return NewBillingHandler(authorizer, service, accounts) +} + +type fixedSessionResolver struct{ session identity.Session } + +func (r *fixedSessionResolver) Resolve(context.Context, string) (identity.Session, error) { + return r.session, nil +} + +func serveJSON(t *testing.T, h http.Handler, method, path string, body any) *httptest.ResponseRecorder { + t.Helper() + var raw []byte + if body != nil { + raw, _ = json.Marshal(body) + } + req := httptest.NewRequest(method, path, bytes.NewReader(raw)) + req.AddCookie(&http.Cookie{Name: identity.SessionCookieName, Value: "signed"}) + response := httptest.NewRecorder() + h.ServeHTTP(response, req) + return response +} diff --git a/backend/internal/httpapi/httpapi.go b/backend/internal/httpapi/httpapi.go index 6e67e37..d6cce6a 100644 --- a/backend/internal/httpapi/httpapi.go +++ b/backend/internal/httpapi/httpapi.go @@ -29,6 +29,19 @@ type Readiness interface { type handler struct { readiness Readiness readinessTimeout time.Duration + healthDetails HealthDetails +} + +// HealthDetails contains runtime compatibility fields assembled by the +// application. Keeping this as injected data prevents the transport layer +// from coupling itself to process environment configuration. +type HealthDetails struct { + VisualAPIMode string `json:"visualApiMode"` + EvolinkMode string `json:"evolinkMode"` + SeedanceMode string `json:"seedanceMode"` + BailianMode string `json:"bailianMode"` + AuthMode string `json:"authMode"` + Capabilities []any `json:"capabilities"` } // Option configures the HTTP handler. @@ -43,9 +56,20 @@ func WithReadinessTimeout(timeout time.Duration) Option { } } +// WithHealthDetails injects the provider, authentication, and capability +// summary exposed by the TypeScript-compatible health contract. +func WithHealthDetails(details HealthDetails) Option { + return func(h *handler) { + h.healthDetails = details + if h.healthDetails.Capabilities == nil { + h.healthDetails.Capabilities = []any{} + } + } +} + // NewHandler returns the foundation health/readiness HTTP handler. func NewHandler(readiness Readiness, options ...Option) http.Handler { - h := &handler{readiness: readiness, readinessTimeout: defaultReadinessTimeout} + h := &handler{readiness: readiness, readinessTimeout: defaultReadinessTimeout, healthDetails: HealthDetails{Capabilities: []any{}}} for _, option := range options { option(h) } @@ -73,15 +97,17 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { status = h.readiness.Status() } writeJSON(w, http.StatusOK, struct { - OK bool `json:"ok"` - AppID string `json:"appId"` - WebOnly bool `json:"webOnly"` + OK bool `json:"ok"` + AppID string `json:"appId"` + WebOnly bool `json:"webOnly"` + HealthDetails Database DatabaseStatus `json:"database"` }{ - OK: true, - AppID: appID, - WebOnly: true, - Database: status, + OK: true, + AppID: appID, + WebOnly: true, + HealthDetails: h.healthDetails, + Database: status, }) } diff --git a/backend/internal/httpapi/httpapi_test.go b/backend/internal/httpapi/httpapi_test.go index 00e0b6a..b74b246 100644 --- a/backend/internal/httpapi/httpapi_test.go +++ b/backend/internal/httpapi/httpapi_test.go @@ -43,6 +43,44 @@ func TestHealthReportsProcessAndDatabaseConfigurationWithoutReadinessProbe(t *te } } +func TestHealthIncludesInjectedRuntimeCompatibilityDetails(t *testing.T) { + details := httpapi.HealthDetails{ + VisualAPIMode: "volcengine", + EvolinkMode: "mock", + SeedanceMode: "seedance", + BailianMode: "missing", + AuthMode: "configured", + Capabilities: []any{ + map[string]any{"id": "image.generate", "engine": "jimeng"}, + map[string]any{"id": "video.generate", "engine": "seedance"}, + }, + } + recorder := httptest.NewRecorder() + + httpapi.NewHandler(&readinessStub{}, httpapi.WithHealthDetails(details)).ServeHTTP( + recorder, + httptest.NewRequest(http.MethodGet, "/api/health", nil), + ) + + var response struct { + VisualAPIMode string `json:"visualApiMode"` + EvolinkMode string `json:"evolinkMode"` + SeedanceMode string `json:"seedanceMode"` + BailianMode string `json:"bailianMode"` + AuthMode string `json:"authMode"` + Capabilities []map[string]any `json:"capabilities"` + } + if err := json.NewDecoder(recorder.Body).Decode(&response); err != nil { + t.Fatalf("decode response: %v", err) + } + if response.VisualAPIMode != "volcengine" || response.EvolinkMode != "mock" || response.SeedanceMode != "seedance" || response.BailianMode != "missing" || response.AuthMode != "configured" { + t.Fatalf("runtime response = %+v", response) + } + if len(response.Capabilities) != 2 || response.Capabilities[0]["id"] != "image.generate" || response.Capabilities[1]["id"] != "video.generate" { + t.Fatalf("capabilities = %#v", response.Capabilities) + } +} + func TestReadyReportsSuccessfulDatabaseProbe(t *testing.T) { readiness := &readinessStub{ status: httpapi.DatabaseStatus{Backend: "postgres", Configured: true}, diff --git a/backend/internal/httpapi/jobs.go b/backend/internal/httpapi/jobs.go new file mode 100644 index 0000000..abbf979 --- /dev/null +++ b/backend/internal/httpapi/jobs.go @@ -0,0 +1,466 @@ +package httpapi + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/orchestration" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/publicapi" +) + +type JobsPublicAuthenticator interface { + Authenticate(*http.Request) (publicapi.PublicClient, string, error) + AssertInternalWorker(*http.Request) error +} +type JobBuildInput struct { + Scope jobs.Scope + Capability string + Body map[string]any + IdempotencyKey string + Origin string +} +type JobBuilder interface { + Build(context.Context, JobBuildInput) (jobs.CreateCommand, error) +} + +type JobCreationCoordinator interface { + CreatePlatform(context.Context, identity.Session, orchestration.CreationInput) (jobs.Job, bool, error) + CreatePublic(context.Context, orchestration.CreationInput) (jobs.Job, bool, error) +} + +type PlatformJobRetryCoordinator interface { + RetryPlatform(context.Context, identity.Session, jobs.Job) (jobs.Job, error) +} + +type JobTickLimiter interface { + TickLimit(context.Context, string, int) (jobs.TickResult, error) +} + +// ProviderBuilderAdapter exposes jobs.ProviderJobBuilder at the HTTP seam. +type ProviderBuilderAdapter struct{ Builder jobs.ProviderJobBuilder } + +func (a ProviderBuilderAdapter) Build(ctx context.Context, in JobBuildInput) (jobs.CreateCommand, error) { + return a.Builder.Build(ctx, in.Scope.OwnerID, in.Scope.ExternalClientID, in.Capability, in.IdempotencyKey, in.Body) +} + +type JobsDependencies struct { + Service *jobs.Service + Platform *PlatformAuthorizer + Public JobsPublicAuthenticator + Builder JobBuilder + Creation JobCreationCoordinator + Refunds jobs.RefundPort + Artifacts jobs.ArtifactDeleter + Worker JobTickLimiter +} +type JobsConfig struct { + MaxJSONBytes int64 + NewID func() string +} +type jobsHandler struct { + dependencies JobsDependencies + config JobsConfig +} + +func NewJobsHandler(d JobsDependencies, c JobsConfig) (http.Handler, error) { + if d.Service == nil || d.Platform == nil || d.Public == nil || d.Builder == nil { + return nil, errors.New("jobs HTTP dependencies are not configured") + } + if c.MaxJSONBytes <= 0 { + c.MaxJSONBytes = 1 << 20 + } + if c.NewID == nil { + c.NewID = randomJobID + } + return &jobsHandler{d, c}, nil +} + +func (h *jobsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + route, id := matchJobRoute(r.URL.Path) + if route == "" { + http.NotFound(w, r) + return + } + allow := jobAllow(route) + if r.Method == http.MethodOptions { + w.Header().Set("Allow", allow) + w.WriteHeader(http.StatusNoContent) + return + } + if !methodAllowed(allow, r.Method) { + w.Header().Set("Allow", allow) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch route { + case "platform-image-list": + h.platformCollection(w, r, "image.generate") + case "platform-video-list": + h.platformCollection(w, r, "video.generate") + case "platform-image-item": + h.platformItem(w, r, id, false) + case "platform-video-item": + h.platformItem(w, r, id, true) + case "platform-retry": + h.platformRetry(w, r, id) + case "public-list": + h.publicCollection(w, r) + case "public-item": + h.publicGet(w, r, id) + case "public-cancel": + h.publicCancel(w, r, id) + case "worker": + h.workerTick(w, r) + } +} + +func (h *jobsHandler) platformCollection(w http.ResponseWriter, r *http.Request, capability string) { + session, err := h.dependencies.Platform.Authorize(r, PlatformApp) + if err != nil { + writeJobError(w, err, false) + return + } + scope := jobs.Scope{OwnerID: session.User.ID} + if r.Method == http.MethodGet { + items, err := h.dependencies.Service.List(r.Context(), jobs.ListFilter{Scope: scope, Capability: capability, Limit: 200}) + if err != nil { + writeJobError(w, err, false) + return + } + if items == nil { + items = []jobs.Job{} + } + writeJSON(w, 200, map[string]any{"jobs": items}) + return + } + h.create(w, r, scope, capability, false, &session) +} +func (h *jobsHandler) platformItem(w http.ResponseWriter, r *http.Request, id string, video bool) { + session, err := h.dependencies.Platform.Authorize(r, PlatformApp) + if err != nil { + writeJobError(w, err, false) + return + } + scope := jobs.Scope{OwnerID: session.User.ID} + if r.Method == http.MethodGet { + j, err := h.dependencies.Service.Get(r.Context(), scope, id) + if err != nil { + writeJobError(w, err, false) + return + } + writeJSON(w, 200, map[string]any{"job": j}) + return + } + j, err := h.dependencies.Service.Get(r.Context(), scope, id) + if err != nil || ((j.Capability == "video.generate") != video) { + writeJSON(w, 404, map[string]string{"error": "任务不存在"}) + return + } + if !j.Status.Terminal() { + j, err = h.dependencies.Service.Cancel(r.Context(), scope, id, h.dependencies.Refunds) + if err != nil { + writeJobError(w, err, false) + return + } + } + deletedAssets := []string{} + if h.dependencies.Artifacts != nil { + deletedAssets, err = h.dependencies.Artifacts.DeleteOutputs(r.Context(), j) + if err != nil { + writeJobError(w, err, false) + return + } + } + _, err = h.dependencies.Service.Delete(r.Context(), scope, id, nil) + if err != nil { + writeJobError(w, err, false) + return + } + writeJSON(w, 200, map[string]any{"ok": true, "deletedJobId": id, "deletedAssetIds": deletedAssets}) +} +func (h *jobsHandler) platformRetry(w http.ResponseWriter, r *http.Request, id string) { + session, err := h.dependencies.Platform.Authorize(r, PlatformApp) + if err != nil { + writeJobError(w, err, false) + return + } + original, err := h.dependencies.Service.Get(r.Context(), jobs.Scope{OwnerID: session.User.ID}, id) + if err != nil || original.Capability != "image.generate" { + writeJSON(w, 404, map[string]string{"error": "任务不存在"}) + return + } + retryCoordinator, ok := h.dependencies.Creation.(PlatformJobRetryCoordinator) + if !ok { + writeJSON(w, 500, map[string]string{"error": "Internal server error."}) + return + } + j, err := retryCoordinator.RetryPlatform(r.Context(), session, original) + if err != nil { + writeJobError(w, err, false) + return + } + writeJSON(w, 202, map[string]any{"job": j}) +} + +func (h *jobsHandler) publicCollection(w http.ResponseWriter, r *http.Request) { + client, owner, err := h.dependencies.Public.Authenticate(r) + if err != nil { + writeJobError(w, err, true) + return + } + scope := jobs.Scope{OwnerID: owner, ExternalClientID: client.ID} + if r.Method == http.MethodPost { + h.create(w, r, scope, "", true, nil) + return + } + q := r.URL.Query() + limit, _ := strconv.Atoi(q.Get("limit")) + var before *time.Time + if raw := q.Get("before"); raw != "" { + if parsed, e := time.Parse(time.RFC3339, raw); e == nil { + before = &parsed + } + } + items, err := h.dependencies.Service.List(r.Context(), jobs.ListFilter{Scope: scope, Status: jobs.Status(q.Get("status")), Capability: q.Get("capability"), Limit: limit, Before: before}) + if err != nil { + writeJobError(w, err, true) + return + } + if items == nil { + items = []jobs.Job{} + } + writeJSON(w, 200, map[string]any{"jobs": items}) +} +func (h *jobsHandler) publicGet(w http.ResponseWriter, r *http.Request, id string) { + _, scope, ok := h.publicScope(w, r) + if !ok { + return + } + j, err := h.dependencies.Service.Get(r.Context(), scope, id) + if err != nil { + writeJobError(w, err, true) + return + } + writeJSON(w, 200, map[string]any{"job": j}) +} +func (h *jobsHandler) publicCancel(w http.ResponseWriter, r *http.Request, id string) { + _, scope, ok := h.publicScope(w, r) + if !ok { + return + } + j, err := h.dependencies.Service.Cancel(r.Context(), scope, id, h.dependencies.Refunds) + if err != nil { + writeJobError(w, err, true) + return + } + writeJSON(w, 200, map[string]any{"job": j}) +} +func (h *jobsHandler) publicScope(w http.ResponseWriter, r *http.Request) (publicapi.PublicClient, jobs.Scope, bool) { + c, o, e := h.dependencies.Public.Authenticate(r) + if e != nil { + writeJobError(w, e, true) + return publicapi.PublicClient{}, jobs.Scope{}, false + } + return c, jobs.Scope{OwnerID: o, ExternalClientID: c.ID}, true +} +func (h *jobsHandler) create(w http.ResponseWriter, r *http.Request, scope jobs.Scope, capability string, public bool, session *identity.Session) { + body := map[string]any{} + if !decodeJobJSON(w, r, h.config.MaxJSONBytes, &body, public) { + return + } + if capability == "" { + capability = stringValueHTTP(body["capability"]) + if capability == "" { + capability = "image.generate" + } + } + if priority, exists := body["priority"]; exists { + body["priority"] = float64(jobs.NormalizePriority(jobHTTPInt(priority))) + } + if public { + if webhook := strings.TrimSpace(stringValueHTTP(body["webhookUrl"])); webhook != "" { + parsed, parseErr := url.Parse(webhook) + if parseErr != nil || !parsed.IsAbs() || parsed.Host == "" || parsed.User != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "webhookUrl must be an HTTP or HTTPS URL."}) + return + } + body["webhookUrl"] = webhook + } + } + key := strings.TrimSpace(r.Header.Get("Idempotency-Key")) + if key == "" { + key = stringValueHTTP(body["idempotencyKey"]) + } + var created jobs.Job + var reused bool + var err error + if h.dependencies.Creation != nil { + input := orchestration.CreationInput{OwnerID: scope.OwnerID, ExternalClientID: scope.ExternalClientID, Capability: capability, Body: body, IdempotencyKey: key} + if public { + created, reused, err = h.dependencies.Creation.CreatePublic(r.Context(), input) + } else if session != nil { + created, reused, err = h.dependencies.Creation.CreatePlatform(r.Context(), *session, input) + } else { + err = errors.New("platform creation session is unavailable") + } + } else { + cmd, buildErr := h.dependencies.Builder.Build(r.Context(), JobBuildInput{Scope: scope, Capability: capability, Body: body, IdempotencyKey: key, Origin: absoluteRequestURL(r)}) + if buildErr != nil { + writeJobError(w, buildErr, public) + return + } + created, reused, err = h.dependencies.Service.Create(r.Context(), cmd) + } + if err != nil { + writeJobError(w, err, public) + return + } + status := 202 + if reused { + status = 200 + } + response := map[string]any{"job": created} + if public { + response["reused"] = reused + } + writeJSON(w, status, response) +} + +func jobHTTPInt(value any) int { + switch typed := value.(type) { + case float64: + return int(typed) + case int: + return typed + case json.Number: + parsed, _ := strconv.Atoi(typed.String()) + return parsed + default: + return 0 + } +} +func (h *jobsHandler) workerTick(w http.ResponseWriter, r *http.Request) { + if err := h.dependencies.Public.AssertInternalWorker(r); err != nil { + writeJobError(w, err, true) + return + } + if h.dependencies.Worker == nil { + writeJSON(w, 500, map[string]string{"error": "Internal server error."}) + return + } + input := struct { + WorkerID string `json:"workerId"` + Limit int `json:"limit"` + }{} + if !decodeJobJSON(w, r, h.config.MaxJSONBytes, &input, true) { + return + } + if input.WorkerID == "" { + input.WorkerID = h.config.NewID() + } + result, err := h.dependencies.Worker.TickLimit(r.Context(), input.WorkerID, input.Limit) + if err != nil { + writeJobError(w, err, true) + return + } + writeJSON(w, 200, result) +} + +func matchJobRoute(p string) (string, string) { + switch p { + case "/api/generations/image": + return "platform-image-list", "" + case "/api/generations/video": + return "platform-video-list", "" + case "/api/v1/jobs": + return "public-list", "" + case "/api/internal/worker/tick": + return "worker", "" + } + parts := strings.Split(strings.Trim(p, "/"), "/") + if len(parts) == 4 && parts[0] == "api" && parts[1] == "generations" && (parts[2] == "image" || parts[2] == "video") { + return "platform-" + parts[2] + "-item", parts[3] + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "generations" && parts[2] == "image" && parts[4] == "retry" { + return "platform-retry", parts[3] + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "jobs" && parts[4] == "cancel" { + return "public-cancel", parts[3] + } + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "jobs" { + return "public-item", parts[3] + } + return "", "" +} +func jobAllow(route string) string { + switch route { + case "platform-image-list", "platform-video-list", "public-list": + return "GET, POST" + case "platform-image-item", "platform-video-item": + return "GET, DELETE" + case "public-item": + return "GET" + default: + return "POST" + } +} +func decodeJobJSON(w http.ResponseWriter, r *http.Request, limit int64, target any, public bool) bool { + r.Body = http.MaxBytesReader(w, r.Body, limit) + if err := json.NewDecoder(r.Body).Decode(target); err != nil { + message := "Invalid request body." + if !public { + message = "请求参数无效" + } + writeJSON(w, 400, map[string]string{"error": message}) + return false + } + return true +} +func writeJobError(w http.ResponseWriter, err error, public bool) { + var domain *jobs.Error + if errors.As(err, &domain) { + message := domain.Message + if !public && domain.Kind == jobs.ErrorNotFound { + message = "Generation job not found." + } + writeJSON(w, domain.Status, map[string]string{"error": message}) + return + } + var auth *publicapi.AuthError + if errors.As(err, &auth) { + writeJSON(w, auth.Status, map[string]string{"error": auth.Message}) + return + } + var platform *PlatformAuthError + if errors.As(err, &platform) { + writeJSON(w, platform.Status, map[string]string{"error": platform.Message}) + return + } + var billingStatus *billing.StatusError + if errors.As(err, &billingStatus) && (billingStatus.Status == 402 || billingStatus.Status == 409) { + message := "生成任务计费失败。" + if public { + message = "Generation job billing failed." + } + writeJSON(w, billingStatus.Status, map[string]string{"error": message}) + return + } + writeJSON(w, 500, map[string]string{"error": "Internal server error."}) +} +func randomJobID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return "job-" + hex.EncodeToString(b) +} +func stringValueHTTP(v any) string { s, _ := v.(string); return strings.TrimSpace(s) } diff --git a/backend/internal/httpapi/jobs_test.go b/backend/internal/httpapi/jobs_test.go new file mode 100644 index 0000000..c369eea --- /dev/null +++ b/backend/internal/httpapi/jobs_test.go @@ -0,0 +1,326 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/orchestration" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/publicapi" +) + +type httpJobStore struct { + values map[string]jobs.Job + claimed []jobs.Job +} + +func (s *httpJobStore) ListJobs(_ context.Context, f jobs.ListFilter) ([]jobs.Job, error) { + out := []jobs.Job{} + for _, j := range s.values { + if f.Scope.Owns(j) && (f.Capability == "" || j.Capability == f.Capability) && (f.Status == "" || j.Status == f.Status) { + out = append(out, j) + } + } + return out, nil +} +func (s *httpJobStore) FindJob(_ context.Context, sc jobs.Scope, id string) (jobs.Job, bool, error) { + j, ok := s.values[id] + return j, ok && sc.Owns(j), nil +} +func (s *httpJobStore) FindIdempotentJob(_ context.Context, sc jobs.Scope, key string) (jobs.Job, bool, error) { + for _, j := range s.values { + if sc.Owns(j) && j.IdempotencyKey == key { + return j, true, nil + } + } + return jobs.Job{}, false, nil +} +func (s *httpJobStore) CreateJob(_ context.Context, j jobs.Job) (jobs.Job, error) { + s.values[j.ID] = j + return j, nil +} +func (s *httpJobStore) UpdateJob(_ context.Context, id string, p jobs.Patch) (jobs.Job, error) { + j := s.values[id] + if p.Status != nil { + j.Status = *p.Status + } + if p.CompletedAt != nil { + j.CompletedAt = p.CompletedAt + } + if p.ClearLease { + j.LockedAt = nil + j.LockedBy = "" + } + s.values[id] = j + return j, nil +} +func (s *httpJobStore) DeleteJob(_ context.Context, id string) error { + delete(s.values, id) + return nil +} +func (s *httpJobStore) ClaimJobs(context.Context, string, int, int) ([]jobs.Job, error) { + return s.claimed, nil +} + +type jobBuilderStub struct{ next int } + +func (b *jobBuilderStub) Build(_ context.Context, input JobBuildInput) (jobs.CreateCommand, error) { + b.next++ + raw, _ := json.Marshal(input.Body) + return jobs.CreateCommand{Job: jobs.Job{ID: "new-" + string(rune('0'+b.next)), OwnerID: input.Scope.OwnerID, ExternalClientID: input.Scope.ExternalClientID, Capability: input.Capability, Provider: "mock", ReqKey: "fixture", Status: jobs.StatusQueued, Prompt: "hello", RequestPayload: raw, IdempotencyKey: input.IdempotencyKey}, IdempotencyBody: input.Body}, nil +} + +func newJobsHTTP(t *testing.T) (http.Handler, *httpJobStore) { + t.Helper() + store := &httpJobStore{values: map[string]jobs.Job{}} + platform, _ := NewPlatformAuthorizer(AuthState{}, nil) + service := jobs.NewService(store, func() time.Time { return time.Date(2026, 8, 13, 9, 0, 0, 0, time.UTC) }) + builder := &jobBuilderStub{} + creation := &passthroughCreationCoordinator{service: service, builder: builder} + h, err := NewJobsHandler(JobsDependencies{Service: service, Platform: platform, Public: publicapi.NewAuthenticator(publicapi.Config{APIKeys: "agent-a:secret", InternalWorkerToken: "worker-secret", Production: true}), Builder: builder, Creation: creation}, JobsConfig{}) + if err != nil { + t.Fatal(err) + } + return h, store +} + +func TestJobsHTTPPlatformImageVideoLifecycle(t *testing.T) { + h, store := newJobsHTTP(t) + for _, path := range []string{"/api/generations/image", "/api/generations/video"} { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(`{"prompt":"hello"}`)) + h.ServeHTTP(w, r) + if w.Code != 202 { + t.Fatalf("POST %s=%d %s", path, w.Code, w.Body.String()) + } + } + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/generations/image", nil)) + if w.Code != 200 { + t.Fatal(w.Code) + } + store.values["failed"] = jobs.Job{ID: "failed", OwnerID: "demo-merchant", Capability: "image.generate", Provider: "mock", ReqKey: "fixture", Status: jobs.StatusFailed, RequestPayload: []byte(`{}`)} + w = httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/generations/image/failed/retry", nil)) + if w.Code != 202 { + t.Fatalf("retry=%d %s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, "/api/generations/image/failed", nil)) + if w.Code != 200 { + t.Fatalf("delete=%d", w.Code) + } +} + +func TestJobsHTTPPlatformRetryUsesFullCreationCoordinatorForActiveOwnedImage(t *testing.T) { + store := &httpJobStore{values: map[string]jobs.Job{"active": {ID: "active", OwnerID: "demo-merchant", Capability: "image.generate", Status: jobs.StatusRunning}}} + platform, _ := NewPlatformAuthorizer(AuthState{}, nil) + creation := &retryCreationCoordinatorStub{creationCoordinatorStub: creationCoordinatorStub{job: jobs.Job{ID: "fresh", RetryOf: "active", Status: jobs.StatusQueued}}} + h, err := NewJobsHandler(JobsDependencies{Service: jobs.NewService(store, time.Now), Platform: platform, Public: publicapi.NewAuthenticator(publicapi.Config{APIKeys: "agent-a:secret"}), Builder: &jobBuilderStub{}, Creation: creation}, JobsConfig{}) + if err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/generations/image/active/retry", nil)) + if w.Code != 202 || creation.retryCalls != 1 || creation.original.ID != "active" { + t.Fatalf("retry=%d %s calls=%d original=%#v", w.Code, w.Body.String(), creation.retryCalls, creation.original) + } +} + +func TestJobsHTTPPublicScopeIdempotencyCancelAndMethods(t *testing.T) { + h, store := newJobsHTTP(t) + body := `{"capability":"image.generate","prompt":"hello"}` + request := func(method, path string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, path, bytes.NewBufferString(body)) + r.Header.Set("Authorization", "Bearer secret") + r.Header.Set("Idempotency-Key", "same") + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w + } + if w := request(http.MethodPost, "/api/v1/jobs"); w.Code != 202 { + t.Fatalf("create=%d %s", w.Code, w.Body.String()) + } + if w := request(http.MethodPost, "/api/v1/jobs"); w.Code != 200 { + t.Fatalf("replay=%d %s", w.Code, w.Body.String()) + } + store.values["other"] = jobs.Job{ID: "other", OwnerID: "api:agent-b", ExternalClientID: "agent-b", Status: jobs.StatusRunning} + if w := request(http.MethodGet, "/api/v1/jobs/other"); w.Code != 404 { + t.Fatalf("cross scope=%d", w.Code) + } + id := "new-1" + if w := request(http.MethodPost, "/api/v1/jobs/"+id+"/cancel"); w.Code != 200 { + t.Fatalf("cancel=%d %s", w.Code, w.Body.String()) + } + if w := request(http.MethodDelete, "/api/v1/jobs"); w.Code != 405 || w.Header().Get("Allow") != "GET, POST" { + t.Fatalf("method=%d allow=%q", w.Code, w.Header().Get("Allow")) + } +} + +func TestJobsHTTPPublicRejectsRelativeWebhookURL(t *testing.T) { + h, _ := newJobsHTTP(t) + for _, webhook := range []string{"/internal/callback", "ftp://example.test/hook", "https://user:pass@example.test/hook"} { + r := httptest.NewRequest(http.MethodPost, "/api/v1/jobs", bytes.NewBufferString(`{"capability":"image.generate","prompt":"hello","webhookUrl":"`+webhook+`"}`)) + r.Header.Set("Authorization", "Bearer secret") + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest { + t.Fatalf("webhook=%q status=%d body=%s", webhook, w.Code, w.Body.String()) + } + } + r := httptest.NewRequest(http.MethodPost, "/api/v1/jobs", bytes.NewBufferString(`{"capability":"image.generate","prompt":"hello","webhookUrl":"https://hooks.example.test/done"}`)) + r.Header.Set("Authorization", "Bearer secret") + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != http.StatusAccepted { + t.Fatalf("valid status=%d body=%s", w.Code, w.Body.String()) + } +} + +func TestJobsHTTPClampsPriorityBeforePassingTrustedBoundary(t *testing.T) { + store := &httpJobStore{values: map[string]jobs.Job{}} + platform, _ := NewPlatformAuthorizer(AuthState{}, nil) + creation := &creationCoordinatorStub{job: jobs.Job{ID: "created", Status: jobs.StatusQueued}} + h, _ := NewJobsHandler(JobsDependencies{Service: jobs.NewService(store, time.Now), Platform: platform, Public: publicapi.NewAuthenticator(publicapi.Config{APIKeys: "agent-a:secret"}), Builder: &jobBuilderStub{}, Creation: creation}, JobsConfig{}) + r := httptest.NewRequest(http.MethodPost, "/api/v1/jobs", bytes.NewBufferString(`{"capability":"image.generate","prompt":"hello","priority":999}`)) + r.Header.Set("Authorization", "Bearer secret") + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != http.StatusAccepted || creation.publicInput.Body["priority"] != float64(100) { + t.Fatalf("status=%d input=%#v", w.Code, creation.publicInput.Body) + } +} + +func TestJobsHTTPWorkerTickUsesWorkerAuth(t *testing.T) { + h, _ := newJobsHTTP(t) + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/internal/worker/tick", bytes.NewBufferString(`{}`))) + if w.Code != 401 { + t.Fatalf("unauthorized=%d", w.Code) + } +} + +func TestJobsHTTPPlatformCreationPassesRefreshedSessionToCoordinator(t *testing.T) { + store := &httpJobStore{values: map[string]jobs.Job{}} + platform, _ := NewPlatformAuthorizer(AuthState{}, nil) + creation := &creationCoordinatorStub{job: jobs.Job{ID: "coordinated", Status: jobs.StatusQueued}} + h, err := NewJobsHandler(JobsDependencies{Service: jobs.NewService(store, time.Now), Platform: platform, Public: publicapi.NewAuthenticator(publicapi.Config{APIKeys: "agent-a:secret"}), Builder: &jobBuilderStub{}, Creation: creation}, JobsConfig{}) + if err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/generations/image", bytes.NewBufferString(`{"prompt":"hello"}`))) + if w.Code != 202 || creation.platform.User.ID != "demo-merchant" || creation.platform.User.OrganizationID != "org-demo" { + t.Fatalf("response=%d %s session=%#v", w.Code, w.Body.String(), creation.platform) + } +} + +func TestJobsHTTPMapsBillingChargeErrorsAndPublicUsesPublicCreation(t *testing.T) { + store := &httpJobStore{values: map[string]jobs.Job{}} + platform, _ := NewPlatformAuthorizer(AuthState{}, nil) + creation := &creationCoordinatorStub{err: &billing.StatusError{Status: 402, Err: billing.ErrInsufficientBalance}} + h, _ := NewJobsHandler(JobsDependencies{Service: jobs.NewService(store, time.Now), Platform: platform, Public: publicapi.NewAuthenticator(publicapi.Config{APIKeys: "agent-a:secret"}), Builder: &jobBuilderStub{}, Creation: creation}, JobsConfig{}) + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/generations/image", bytes.NewBufferString(`{"prompt":"hello"}`))) + if w.Code != 402 || w.Body.String() != "{\"error\":\"生成任务计费失败。\"}\n" { + t.Fatalf("platform=%d %s", w.Code, w.Body.String()) + } + creation.err = nil + r := httptest.NewRequest(http.MethodPost, "/api/v1/jobs", bytes.NewBufferString(`{"prompt":"hello"}`)) + r.Header.Set("Authorization", "Bearer secret") + w = httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != 202 || creation.publicCalls != 1 || creation.platformCalls != 1 { + t.Fatalf("public=%d calls=%d/%d", w.Code, creation.platformCalls, creation.publicCalls) + } +} + +func TestJobsHTTPMapsAllCreationBillingOutcomesWithoutLeakingErrors(t *testing.T) { + tests := []struct { + name string + err error + want int + }{ + {name: "insufficient", err: &billing.StatusError{Status: 402, Err: billing.ErrInsufficientBalance}, want: 402}, + {name: "idempotency", err: &billing.StatusError{Status: 409, Err: billing.ErrIdempotencyConflict}, want: 409}, + {name: "unknown", err: errors.New("postgres password leaked"), want: 500}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + store := &httpJobStore{values: map[string]jobs.Job{}} + platform, _ := NewPlatformAuthorizer(AuthState{}, nil) + creation := &creationCoordinatorStub{err: test.err} + h, _ := NewJobsHandler(JobsDependencies{Service: jobs.NewService(store, time.Now), Platform: platform, Public: publicapi.NewAuthenticator(publicapi.Config{APIKeys: "agent-a:secret"}), Builder: &jobBuilderStub{}, Creation: creation}, JobsConfig{}) + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/generations/image", bytes.NewBufferString(`{"prompt":"hello"}`))) + if w.Code != test.want || strings.Contains(w.Body.String(), "postgres") { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + }) + } +} + +type creationCoordinatorStub struct { + job jobs.Job + err error + platform identity.Session + platformCalls, publicCalls int + publicInput orchestration.CreationInput +} + +type retryCreationCoordinatorStub struct { + creationCoordinatorStub + retryCalls int + original jobs.Job +} + +func (s *retryCreationCoordinatorStub) RetryPlatform(_ context.Context, _ identity.Session, original jobs.Job) (jobs.Job, error) { + s.retryCalls++ + s.original = original + return s.job, s.err +} + +type passthroughCreationCoordinator struct { + service *jobs.Service + builder JobBuilder +} + +func (s *passthroughCreationCoordinator) create(ctx context.Context, scope jobs.Scope, in orchestration.CreationInput) (jobs.Job, bool, error) { + command, err := s.builder.Build(ctx, JobBuildInput{Scope: scope, Capability: in.Capability, Body: in.Body, IdempotencyKey: in.IdempotencyKey}) + if err != nil { + return jobs.Job{}, false, err + } + return s.service.Create(ctx, command) +} +func (s *passthroughCreationCoordinator) CreatePlatform(ctx context.Context, session identity.Session, in orchestration.CreationInput) (jobs.Job, bool, error) { + return s.create(ctx, jobs.Scope{OwnerID: session.User.ID}, in) +} +func (s *passthroughCreationCoordinator) CreatePublic(ctx context.Context, in orchestration.CreationInput) (jobs.Job, bool, error) { + return s.create(ctx, jobs.Scope{OwnerID: in.OwnerID, ExternalClientID: in.ExternalClientID}, in) +} +func (s *passthroughCreationCoordinator) RetryPlatform(ctx context.Context, session identity.Session, original jobs.Job) (jobs.Job, error) { + body := map[string]any{"prompt": original.Prompt} + job, _, err := s.create(ctx, jobs.Scope{OwnerID: session.User.ID}, orchestration.CreationInput{Capability: original.Capability, Body: body}) + job.RetryOf = original.ID + return job, err +} + +func (s *creationCoordinatorStub) CreatePlatform(_ context.Context, session identity.Session, _ orchestration.CreationInput) (jobs.Job, bool, error) { + s.platform = session + s.platformCalls++ + return s.job, false, s.err +} +func (s *creationCoordinatorStub) CreatePublic(_ context.Context, input orchestration.CreationInput) (jobs.Job, bool, error) { + s.publicCalls++ + s.publicInput = input + return s.job, false, s.err +} diff --git a/backend/internal/httpapi/method_compat.go b/backend/internal/httpapi/method_compat.go new file mode 100644 index 0000000..16b2d6c --- /dev/null +++ b/backend/internal/httpapi/method_compat.go @@ -0,0 +1,104 @@ +package httpapi + +import ( + "net/http" + "sort" + "strings" +) + +type routeMethodPattern struct { + path string + methods map[string]struct{} + allow []string +} + +// WithRouteMethodCompatibility applies the method behavior that Next derives +// from app route exports. It sits outside authentication so OPTIONS never +// depends on runtime configuration and HEAD executes the corresponding GET +// semantics while suppressing the response body. +func WithRouteMethodCompatibility(next http.Handler) http.Handler { + patterns := routeMethodPatterns(GoRouteSurface()) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pattern := matchingRouteMethodPattern(patterns, r.URL.Path) + if pattern == nil { + next.ServeHTTP(w, r) + return + } + + if r.Method == http.MethodOptions { + w.Header().Set("Allow", strings.Join(pattern.allow, ", ")) + w.WriteHeader(http.StatusNoContent) + return + } + if r.Method == http.MethodHead { + if _, ok := pattern.methods[http.MethodHead]; !ok { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + clone := r.Clone(r.Context()) + clone.Method = http.MethodGet + next.ServeHTTP(&methodHeadResponseWriter{ResponseWriter: w}, clone) + return + } + if _, ok := pattern.methods[r.Method]; !ok { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + next.ServeHTTP(w, r) + }) +} + +func routeMethodPatterns(surface []RouteSurface) []routeMethodPattern { + byPath := make(map[string]int) + patterns := make([]routeMethodPattern, 0) + for _, route := range surface { + index, exists := byPath[route.Path] + if !exists { + index = len(patterns) + byPath[route.Path] = index + patterns = append(patterns, routeMethodPattern{path: route.Path, methods: make(map[string]struct{})}) + } + patterns[index].methods[route.Method] = struct{}{} + } + for index := range patterns { + if _, hasGet := patterns[index].methods[http.MethodGet]; hasGet { + patterns[index].methods[http.MethodHead] = struct{}{} + } + patterns[index].methods[http.MethodOptions] = struct{}{} + patterns[index].allow = make([]string, 0, len(patterns[index].methods)) + for method := range patterns[index].methods { + patterns[index].allow = append(patterns[index].allow, method) + } + sort.Strings(patterns[index].allow) + } + return patterns +} + +func matchingRouteMethodPattern(patterns []routeMethodPattern, path string) *routeMethodPattern { + for index := range patterns { + if MatchSurfacePath(patterns[index].path, path) { + return &patterns[index] + } + } + return nil +} + +type methodHeadResponseWriter struct { + http.ResponseWriter + wroteHeader bool +} + +func (writer *methodHeadResponseWriter) WriteHeader(status int) { + if writer.wroteHeader { + return + } + writer.wroteHeader = true + writer.ResponseWriter.WriteHeader(status) +} + +func (writer *methodHeadResponseWriter) Write(body []byte) (int, error) { + if !writer.wroteHeader { + writer.WriteHeader(http.StatusOK) + } + return len(body), nil +} diff --git a/backend/internal/httpapi/method_compat_test.go b/backend/internal/httpapi/method_compat_test.go new file mode 100644 index 0000000..1df0f88 --- /dev/null +++ b/backend/internal/httpapi/method_compat_test.go @@ -0,0 +1,118 @@ +package httpapi + +import ( + "io" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "strings" + "testing" +) + +func TestRouteMethodCompatibilityDerivesEverySurfacePath(t *testing.T) { + patterns := routeMethodPatterns(GoRouteSurface()) + if len(patterns) != 47 { + t.Fatalf("route patterns=%d want 47", len(patterns)) + } + for _, pattern := range patterns { + if _, ok := pattern.methods[http.MethodOptions]; !ok { + t.Fatalf("%s omits OPTIONS", pattern.path) + } + if _, hasGet := pattern.methods[http.MethodGet]; hasGet { + if _, hasHead := pattern.methods[http.MethodHead]; !hasHead { + t.Fatalf("%s omits derived HEAD", pattern.path) + } + } + if !sort.StringsAreSorted(pattern.allow) { + t.Fatalf("%s Allow is not sorted: %v", pattern.path, pattern.allow) + } + } +} + +func TestRouteMethodCompatibilityHandlesOptionsBeforeApplicationAuth(t *testing.T) { + calls := 0 + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ }) + handler := WithRouteMethodCompatibility(next) + + for _, test := range []struct { + path string + allow string + }{ + {path: "/api/health", allow: "GET, HEAD, OPTIONS"}, + {path: "/api/admin/accounts", allow: "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT"}, + {path: "/api/v1/jobs", allow: "GET, HEAD, OPTIONS, POST"}, + {path: "/api/v1/jobs/job-1/cancel", allow: "OPTIONS, POST"}, + {path: "/uploads/2026/08/file.png", allow: "GET, HEAD, OPTIONS"}, + } { + t.Run(test.path, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodOptions, test.path, nil)) + if response.Code != http.StatusNoContent || response.Body.Len() != 0 { + t.Fatalf("status=%d body=%q", response.Code, response.Body.String()) + } + if got := response.Header().Get("Allow"); got != test.allow { + t.Fatalf("Allow=%q want %q", got, test.allow) + } + }) + } + if calls != 0 { + t.Fatalf("OPTIONS reached application %d times", calls) + } +} + +func TestRouteMethodCompatibilityDerivesHeadAndSuppressesBody(t *testing.T) { + var methods []string + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{"secret":"must-not-be-written"}`) + }) + handler := WithRouteMethodCompatibility(next) + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodHead, "/api/v1/assets/asset-1/download", nil)) + if response.Code != http.StatusCreated || response.Body.Len() != 0 { + t.Fatalf("status=%d body=%q", response.Code, response.Body.String()) + } + if response.Header().Get("Content-Type") != "application/json" { + t.Fatalf("content-type=%q", response.Header().Get("Content-Type")) + } + if !reflect.DeepEqual(methods, []string{http.MethodGet}) { + t.Fatalf("downstream methods=%v", methods) + } +} + +func TestRouteMethodCompatibilityReturnsFrameworkStyle405AndPassesUnknownPaths(t *testing.T) { + calls := 0 + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + http.NotFound(w, r) + }) + handler := WithRouteMethodCompatibility(next) + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodDelete, "/api/health", strings.NewReader("ignored"))) + if response.Code != http.StatusMethodNotAllowed || response.Body.Len() != 0 { + t.Fatalf("status=%d body=%q", response.Code, response.Body.String()) + } + if got := response.Header().Get("Allow"); got != "" { + t.Fatalf("unsupported method unexpectedly exposes Allow=%q", got) + } + if calls != 0 { + t.Fatalf("unsupported method reached application") + } + + response = httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodOptions, "/not-a-route", nil)) + if response.Code != http.StatusNotFound || calls != 1 { + t.Fatalf("unknown status=%d calls=%d", response.Code, calls) + } + + response = httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodOptions, "/uploads/", nil)) + if response.Code != http.StatusNotFound || calls != 2 { + t.Fatalf("empty catch-all status=%d calls=%d", response.Code, calls) + } +} diff --git a/backend/internal/httpapi/misc.go b/backend/internal/httpapi/misc.go new file mode 100644 index 0000000..e37c181 --- /dev/null +++ b/backend/internal/httpapi/misc.go @@ -0,0 +1,336 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/prompt" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/publicapi" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates" +) + +type SettingsService interface { + Get(context.Context) (any, error) + Save(context.Context, map[string]any) (any, error) +} +type LogFilters struct { + Level, Q, Source string + Limit int +} +type LogService interface { + List(context.Context, LogFilters) (any, error) + Clear(context.Context) error +} +type PublicRequestAuthenticator interface { + Authenticate(*http.Request) (publicapi.PublicClient, string, error) +} +type MiscDependencies struct { + Platform *PlatformAuthorizer + Templates *templates.Service + PromptAssembler func(prompt.Input) prompt.Result + Settings SettingsService + Logs LogService + Public PublicRequestAuthenticator + Capabilities func(context.Context) (any, error) + PublicOrigin string +} +type miscHandler struct{ dependencies MiscDependencies } + +func NewMiscHandler(dependencies MiscDependencies) (http.Handler, error) { + if dependencies.Platform == nil { + return nil, fmt.Errorf("misc HTTP: platform authorizer is required") + } + if dependencies.PromptAssembler == nil { + dependencies.PromptAssembler = prompt.Assemble + } + return &miscHandler{dependencies: dependencies}, nil +} + +func (handler *miscHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/image-templates" || strings.HasPrefix(r.URL.Path, "/api/image-templates/"): + handler.templates(w, r) + case r.URL.Path == "/api/prompt/assemble": + handler.assemble(w, r) + case r.URL.Path == "/api/settings": + handler.settings(w, r) + case r.URL.Path == "/api/logs": + handler.logs(w, r) + case r.URL.Path == "/api/v1/capabilities": + handler.capabilities(w, r) + case r.URL.Path == "/api/v1/openapi.json": + handler.openapi(w, r) + default: + http.NotFound(w, r) + } +} + +func (handler *miscHandler) templates(w http.ResponseWriter, r *http.Request) { + if handler.dependencies.Templates == nil { + writeMiscError(w, 500) + return + } + session, err := handler.dependencies.Platform.Authorize(r, PlatformApp) + if err != nil { + writeMiscAuth(w, err) + return + } + id := strings.TrimPrefix(r.URL.Path, "/api/image-templates/") + collection := r.URL.Path == "/api/image-templates" + if collection && r.Method == http.MethodGet { + items, current := handler.dependencies.Templates.List(r.Context(), session.User.ID) + if current != nil { + writeMiscError(w, 500) + return + } + writeJSON(w, 200, map[string]any{"templates": items}) + return + } + if collection && r.Method == http.MethodPost { + var command templates.CreateCommand + if !decodeMiscJSON(w, r, &command) { + return + } + item, current := handler.dependencies.Templates.Create(r.Context(), session.User.ID, command) + if current != nil { + writeTemplateError(w, current) + return + } + writeJSON(w, 201, map[string]any{"template": item}) + return + } + if !collection && id != "" && r.Method == http.MethodPatch { + var command templates.UpdateCommand + if !decodeMiscJSON(w, r, &command) { + return + } + item, current := handler.dependencies.Templates.Update(r.Context(), session.User.ID, id, command) + if current != nil { + writeTemplateError(w, current) + return + } + writeJSON(w, 200, map[string]any{"template": item}) + return + } + if !collection && id != "" && r.Method == http.MethodDelete { + item, current := handler.dependencies.Templates.Delete(r.Context(), session.User.ID, id) + if current != nil { + writeTemplateError(w, current) + return + } + writeJSON(w, 200, map[string]any{"template": item}) + return + } + writeMiscMethod(w, allowedTemplateMethods(collection)) +} +func (handler *miscHandler) assemble(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeMiscMethod(w, "POST") + return + } + if _, err := handler.dependencies.Platform.Authorize(r, PlatformApp); err != nil { + writeMiscAuth(w, err) + return + } + var input prompt.Input + if !decodeMiscJSON(w, r, &input) { + return + } + writeJSON(w, 200, handler.dependencies.PromptAssembler(input)) +} +func (handler *miscHandler) settings(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodPost { + writeMiscMethod(w, "GET, POST") + return + } + if _, err := handler.dependencies.Platform.Authorize(r, PlatformSuperAdmin); err != nil { + writeMiscAuth(w, err) + return + } + if handler.dependencies.Settings == nil { + writeMiscError(w, 500) + return + } + var ( + value any + err error + ) + if r.Method == http.MethodGet { + value, err = handler.dependencies.Settings.Get(r.Context()) + } else { + var body struct { + Values map[string]any `json:"values"` + } + if !decodeMiscJSON(w, r, &body) { + return + } + if body.Values == nil { + body.Values = map[string]any{} + } + value, err = handler.dependencies.Settings.Save(r.Context(), body.Values) + } + if err != nil { + writeMiscError(w, 500) + return + } + writeJSON(w, 200, value) +} +func (handler *miscHandler) logs(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodDelete { + writeMiscMethod(w, "GET, DELETE") + return + } + if _, err := handler.dependencies.Platform.Authorize(r, PlatformSuperAdmin); err != nil { + writeMiscAuth(w, err) + return + } + if handler.dependencies.Logs == nil { + writeMiscError(w, 500) + return + } + if r.Method == http.MethodDelete { + if err := handler.dependencies.Logs.Clear(r.Context()); err != nil { + writeMiscError(w, 500) + return + } + writeJSON(w, 200, map[string]any{"ok": true}) + return + } + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + if limit == 0 { + limit = 100 + } + level := r.URL.Query().Get("level") + if level != "info" && level != "warning" && level != "error" { + level = "all" + } + entries, err := handler.dependencies.Logs.List(r.Context(), LogFilters{Level: level, Q: r.URL.Query().Get("q"), Limit: limit}) + if err != nil { + writeMiscError(w, 500) + return + } + writeJSON(w, 200, map[string]any{"entries": entries}) +} +func (handler *miscHandler) capabilities(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeMiscMethod(w, "GET") + return + } + if handler.dependencies.Public == nil { + writeMiscError(w, 500) + return + } + if _, _, err := handler.dependencies.Public.Authenticate(r); err != nil { + writePublicMiscError(w, err) + return + } + if handler.dependencies.Capabilities == nil { + writeJSON(w, 200, map[string]any{"capabilities": []any{}}) + return + } + items, err := handler.dependencies.Capabilities(r.Context()) + if err != nil { + writeMiscError(w, 500) + return + } + writeJSON(w, 200, map[string]any{"capabilities": items}) +} +func (handler *miscHandler) openapi(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeMiscMethod(w, "GET") + return + } + origin := normalizeMiscPublicOrigin(handler.dependencies.PublicOrigin) + if origin == "" { + origin = miscRequestOrigin(r) + } + writeJSON(w, 200, openAPIDocument(origin)) +} + +func decodeMiscJSON(w http.ResponseWriter, r *http.Request, destination any) bool { + decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<20)) + if err := decoder.Decode(destination); err != nil && err != io.EOF { + writeJSON(w, 400, map[string]string{"error": "请求参数无效。"}) + return false + } + return true +} +func writeTemplateError(w http.ResponseWriter, err error) { + if errors.Is(err, templates.ErrNotFound) { + writeJSON(w, 404, map[string]string{"error": "模板不存在"}) + return + } + if errors.Is(err, templates.ErrInvalidTemplate) { + writeJSON(w, 400, map[string]string{"error": strings.TrimPrefix(err.Error(), templates.ErrInvalidTemplate.Error()+": ")}) + return + } + writeMiscError(w, 500) +} +func writeMiscAuth(w http.ResponseWriter, err error) { + var auth *PlatformAuthError + if errors.As(err, &auth) { + writeJSON(w, auth.Status, map[string]string{"error": auth.Message}) + return + } + writeMiscError(w, 500) +} +func writePublicMiscError(w http.ResponseWriter, err error) { + var auth *publicapi.AuthError + if errors.As(err, &auth) { + writeJSON(w, auth.Status, map[string]string{"error": auth.Message}) + return + } + writeMiscError(w, 500) +} +func writeMiscError(w http.ResponseWriter, status int) { + writeJSON(w, status, map[string]string{"error": "服务器内部错误。"}) +} +func writeMiscMethod(w http.ResponseWriter, allow string) { + w.Header().Set("Allow", allow) + writeJSON(w, 405, map[string]string{"error": "Method Not Allowed"}) +} +func allowedTemplateMethods(collection bool) string { + if collection { + return "GET, POST" + } + return "PATCH, DELETE" +} +func miscRequestOrigin(r *http.Request) string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + if forwarded := strings.TrimSpace(strings.Split(r.Header.Get("x-forwarded-proto"), ",")[0]); forwarded == "http" || forwarded == "https" { + scheme = forwarded + } + host := r.Host + if parsed, err := url.Parse(scheme + "://" + host); err == nil { + if parsed.Hostname() == "0.0.0.0" { + parsed.Host = strings.Replace(parsed.Host, "0.0.0.0", "127.0.0.1", 1) + } + return parsed.Scheme + "://" + parsed.Host + } + return scheme + "://" + host +} + +func normalizeMiscPublicOrigin(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return strings.TrimSuffix(value, "/") + } + if parsed.Hostname() == "0.0.0.0" { + parsed.Host = strings.Replace(parsed.Host, "0.0.0.0", "127.0.0.1", 1) + } + return parsed.Scheme + "://" + parsed.Host +} diff --git a/backend/internal/httpapi/misc_test.go b/backend/internal/httpapi/misc_test.go new file mode 100644 index 0000000..0634457 --- /dev/null +++ b/backend/internal/httpapi/misc_test.go @@ -0,0 +1,161 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/prompt" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/publicapi" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates" +) + +type miscResolver struct { + session identity.Session + err error +} + +func (resolver miscResolver) Resolve(context.Context, string) (identity.Session, error) { + return resolver.session, resolver.err +} + +type miscTemplateCatalog struct{ items []templates.Template } + +func (c *miscTemplateCatalog) ListTemplates(context.Context, string) ([]templates.Template, error) { + return c.items, nil +} +func (c *miscTemplateCatalog) CreateTemplate(_ context.Context, item templates.Template) (templates.Template, error) { + c.items = append(c.items, item) + return item, nil +} +func (c *miscTemplateCatalog) UpdateTemplate(context.Context, string, string, templates.Patch, time.Time) (templates.Template, bool, error) { + return templates.Template{}, false, nil +} +func (c *miscTemplateCatalog) DeleteTemplate(context.Context, string, string) (templates.Template, bool, error) { + return templates.Template{}, false, nil +} + +func TestMiscTemplateAndPromptRoutes(t *testing.T) { + version := 1 + session := identity.Session{Version: 1, AuthMode: identity.AuthModeUser, SessionVersion: &version, User: identity.User{ID: "u1", ClientID: "platform", Role: "user"}} + authorizer, _ := NewPlatformAuthorizer(AuthState{Required: true, Configured: true}, miscResolver{session: session}) + catalog := &miscTemplateCatalog{} + service := templates.NewService(catalog, func() time.Time { return time.Date(2026, 8, 13, 0, 0, 0, 0, time.UTC) }, func() string { return "t1" }) + handler, err := NewMiscHandler(MiscDependencies{Platform: authorizer, Templates: service, PromptAssembler: prompt.Assemble}) + if err != nil { + t.Fatal(err) + } + post := httptest.NewRequest(http.MethodPost, "/api/image-templates", bytes.NewBufferString(`{"name":" N ","prompt":" P "}`)) + post.AddCookie(&http.Cookie{Name: identity.SessionCookieName, Value: "cookie"}) + w := httptest.NewRecorder() + handler.ServeHTTP(w, post) + if w.Code != http.StatusCreated { + t.Fatalf("create status=%d body=%s", w.Code, w.Body.String()) + } + get := httptest.NewRequest(http.MethodGet, "/api/image-templates", nil) + get.AddCookie(&http.Cookie{Name: identity.SessionCookieName, Value: "cookie"}) + w = httptest.NewRecorder() + handler.ServeHTTP(w, get) + if w.Code != 200 || !bytes.Contains(w.Body.Bytes(), []byte(`"templates"`)) { + t.Fatalf("list status=%d body=%s", w.Code, w.Body.String()) + } + assemble := httptest.NewRequest(http.MethodPost, "/api/prompt/assemble", bytes.NewBufferString(`{"mode":"image","manualPrompt":"@图片1","materials":[]}`)) + assemble.AddCookie(&http.Cookie{Name: identity.SessionCookieName, Value: "cookie"}) + w = httptest.NewRecorder() + handler.ServeHTTP(w, assemble) + if w.Code != 200 || !bytes.Contains(w.Body.Bytes(), []byte(`"requirements":{"image":1`)) { + t.Fatalf("prompt status=%d body=%s", w.Code, w.Body.String()) + } +} + +type settingsStub struct { + read any + saved map[string]any + err error +} + +func (s *settingsStub) Get(context.Context) (any, error) { return s.read, s.err } +func (s *settingsStub) Save(_ context.Context, v map[string]any) (any, error) { + s.saved = v + return s.read, s.err +} + +type logsStub struct { + entries any + cleared bool + err error +} + +func (s *logsStub) List(context.Context, LogFilters) (any, error) { return s.entries, s.err } +func (s *logsStub) Clear(context.Context) error { s.cleared = true; return s.err } + +func TestMiscSettingsLogsCapabilitiesAndOpenAPI(t *testing.T) { + session := identity.Session{AuthMode: identity.AuthModeAdmin, User: identity.User{ID: "root", ClientID: "platform", Role: "super_admin"}} + authorizer, _ := NewPlatformAuthorizer(AuthState{Required: true, Configured: true}, miscResolver{session: session}) + settings := &settingsStub{read: map[string]any{"groups": []any{}}} + logs := &logsStub{entries: []any{map[string]any{"id": "l1"}}} + public := publicapi.NewAuthenticator(publicapi.Config{APIKeys: "client:key"}) + handler, err := NewMiscHandler(MiscDependencies{Platform: authorizer, Settings: settings, Logs: logs, Public: public, Capabilities: func(context.Context) (any, error) { return []any{map[string]any{"id": "image.generate"}}, nil }}) + if err != nil { + t.Fatal(err) + } + for _, path := range []string{"/api/settings", "/api/logs"} { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.AddCookie(&http.Cookie{Name: identity.SessionCookieName, Value: "cookie"}) + w := httptest.NewRecorder() + handler.ServeHTTP(w, request) + if w.Code != 200 { + t.Fatalf("%s status %d %s", path, w.Code, w.Body.String()) + } + } + capRequest := httptest.NewRequest(http.MethodGet, "/api/v1/capabilities", nil) + capRequest.Header.Set("authorization", "Bearer key") + w := httptest.NewRecorder() + handler.ServeHTTP(w, capRequest) + if w.Code != 200 { + t.Fatalf("capabilities %d %s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "https://app.example.test/api/v1/openapi.json", nil)) + if w.Code != 200 || !bytes.Contains(w.Body.Bytes(), []byte(`"openapi":"3.1.0"`)) || !bytes.Contains(w.Body.Bytes(), []byte(`https://app.example.test`)) { + t.Fatalf("openapi %d %s", w.Code, w.Body.String()) + } +} + +func TestMiscOpenAPIUsesConfiguredPublicOrigin(t *testing.T) { + authorizer, _ := NewPlatformAuthorizer(AuthState{}, nil) + handler, err := NewMiscHandler(MiscDependencies{Platform: authorizer, PublicOrigin: "https://public.example.test/base/"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "http://internal:3000/api/v1/openapi.json", nil)) + if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte(`"url":"https://public.example.test"`)) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + +func TestMiscFailsClosedAndHidesInfrastructureErrors(t *testing.T) { + authorizer, _ := NewPlatformAuthorizer(AuthState{Required: true, Configured: true}, miscResolver{err: errors.New("db password secret")}) + handler, _ := NewMiscHandler(MiscDependencies{Platform: authorizer, PromptAssembler: prompt.Assemble}) + request := httptest.NewRequest(http.MethodPost, "/api/prompt/assemble", bytes.NewBufferString(`{"mode":"image"}`)) + request.AddCookie(&http.Cookie{Name: identity.SessionCookieName, Value: "cookie"}) + w := httptest.NewRecorder() + handler.ServeHTTP(w, request) + if w.Code != 500 || bytes.Contains(w.Body.Bytes(), []byte("password")) { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/settings", nil)) + var body map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &body) + if w.Code != 401 { + t.Fatalf("status=%d body=%#v", w.Code, body) + } +} diff --git a/backend/internal/httpapi/openapi.go b/backend/internal/httpapi/openapi.go new file mode 100644 index 0000000..5b7a17c --- /dev/null +++ b/backend/internal/httpapi/openapi.go @@ -0,0 +1,316 @@ +package httpapi + +func openAPIDocument(origin string) map[string]any { + return map[string]any{ + "openapi": "3.1.0", + "info": map[string]any{ + "title": "智念AIGC平台 Public API", + "version": "1.0.0", + "description": "Public server-to-server API for uploading assets, creating image/video generation jobs, polling job status, downloading outputs, and receiving webhooks.", + }, + "servers": []any{map[string]any{"url": origin, "description": "Current deployment"}}, + "security": []any{ + map[string]any{"bearerApiKey": []any{}}, + map[string]any{"headerApiKey": []any{}}, + }, + "components": map[string]any{ + "securitySchemes": map[string]any{ + "bearerApiKey": map[string]any{"type": "http", "scheme": "bearer"}, + "headerApiKey": map[string]any{"type": "apiKey", "in": "header", "name": "X-Zhinian-Api-Key"}, + }, + "schemas": openAPISchemas(), + }, + "paths": openAPIPaths(), + } +} + +func openAPISchemas() map[string]any { + return map[string]any{ + "ErrorResponse": map[string]any{ + "type": "object", "required": []string{"error"}, + "properties": map[string]any{"error": map[string]any{"type": "string"}}, + }, + "Asset": map[string]any{ + "type": "object", + "required": []string{"id", "kind", "name", "url", "source", "createdAt"}, + "properties": map[string]any{ + "id": map[string]any{"type": "string", "example": "asset_mpqe9g85_12635f8cd8"}, + "ownerId": map[string]any{"type": "string"}, + "kind": map[string]any{"type": "string", "enum": []string{"image", "video", "mask", "reference", "other"}}, + "name": map[string]any{"type": "string", "example": "result.png"}, + "url": map[string]any{"type": "string", "format": "uri"}, + "storagePath": map[string]any{"type": "string"}, + "source": map[string]any{"type": "string", "enum": []string{"upload", "generated", "external", "seed"}}, + "tags": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, + "metadata": map[string]any{"type": "object", "additionalProperties": true}, + "createdAt": map[string]any{"type": "string", "format": "date-time"}, + "updatedAt": map[string]any{"type": "string", "format": "date-time"}, + }, + }, + "GenerationJob": map[string]any{ + "type": "object", + "required": []string{"id", "capability", "provider", "status", "createdAt", "updatedAt"}, + "properties": map[string]any{ + "id": map[string]any{"type": "string", "example": "job_mpqe3wtt_12ed738079"}, + "ownerId": map[string]any{"type": "string", "example": "api:partner-a"}, + "externalClientId": map[string]any{"type": "string"}, + "capability": openAPIRef("GenerationCapability"), + "provider": map[string]any{"type": "string", "enum": []string{"volcengine-visual", "evolink", "seedance", "bailian", "mock"}}, + "reqKey": map[string]any{"type": "string"}, + "status": openAPIRef("GenerationStatus"), + "prompt": map[string]any{"type": "string"}, + "inputAssetIds": openAPIStringArray(), + "inputUrls": map[string]any{"type": "array", "items": map[string]any{"type": "string", "format": "uri"}}, + "outputAssetIds": openAPIStringArray(), + "providerTaskId": map[string]any{"type": "string"}, + "error": openAPIJobError(), + "idempotencyKey": map[string]any{"type": "string"}, + "priority": map[string]any{"type": "integer"}, + "attempts": map[string]any{"type": "integer"}, + "scheduledAt": map[string]any{"type": "string", "format": "date-time"}, + "completedAt": map[string]any{"type": "string", "format": "date-time"}, + "webhookUrl": map[string]any{"type": "string", "format": "uri"}, + "webhookAttempts": map[string]any{"type": "integer"}, + "webhookLastStatus": map[string]any{"type": "object", "additionalProperties": true}, + "createdAt": map[string]any{"type": "string", "format": "date-time"}, + "updatedAt": map[string]any{"type": "string", "format": "date-time"}, + }, + }, + "GenerationCapability": map[string]any{"type": "string", "enum": []string{"image.generate", "video.generate"}}, + "GenerationStatus": map[string]any{"type": "string", "enum": []string{"queued", "running", "succeeded", "failed", "expired", "cancelled"}}, + "PromptMaterial": map[string]any{ + "type": "object", + "properties": map[string]any{ + "id": map[string]any{"type": "string"}, + "url": map[string]any{"type": "string", "format": "uri"}, + "type": map[string]any{"type": "string", "enum": []string{"image", "video", "audio"}}, + "role": map[string]any{"type": "string"}, + "label": map[string]any{"type": "string"}, + "name": map[string]any{"type": "string"}, + }, + }, + "CreateJobRequest": map[string]any{ + "type": "object", + "required": []string{"capability"}, + "properties": map[string]any{ + "capability": openAPIRef("GenerationCapability"), + "prompt": map[string]any{"type": "string", "description": "Prompt text. Required for image.generate unless promptAssembly is supplied."}, + "inputUrls": map[string]any{"type": "array", "items": map[string]any{"type": "string", "format": "uri"}, "description": "Reference image URLs for image capabilities."}, + "imageUrls": map[string]any{"type": "array", "items": map[string]any{"type": "string", "format": "uri"}, "description": "Alias for image input URLs."}, + "inputAssetIds": openAPIStringArray(), + "materials": map[string]any{"type": "array", "items": openAPIRef("PromptMaterial")}, + "settings": map[string]any{ + "type": "object", "description": "Video settings for video.generate.", + "properties": map[string]any{ + "ratio": map[string]any{"type": "string", "enum": []string{"16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"}}, + "duration": map[string]any{"type": "integer", "minimum": 4, "maximum": 15}, + "resolution": map[string]any{"type": "string", "enum": []string{"480p", "720p", "1080p"}}, + }, + }, + "width": map[string]any{"type": "integer", "example": 1440}, + "height": map[string]any{"type": "integer", "example": 2560}, + "scale": map[string]any{"type": "number", "minimum": 1, "maximum": 100, "description": "Jimeng text influence for image.generate."}, + "force_single": map[string]any{"type": "boolean"}, + "quality": map[string]any{"type": "string", "enum": []string{"low", "medium", "high"}, "description": "EvoLink image quality for image.generate."}, + "priority": map[string]any{"type": "integer", "minimum": -100, "maximum": 100}, + "webhookUrl": map[string]any{"type": "string", "format": "uri"}, + "idempotencyKey": map[string]any{"type": "string", "description": "Optional body-level idempotency key. Header Idempotency-Key is preferred."}, + }, + }, + "RegisterAssetRequest": map[string]any{ + "type": "object", "required": []string{"url"}, + "properties": map[string]any{ + "url": map[string]any{"type": "string", "format": "uri"}, + "name": map[string]any{"type": "string"}, + "kind": map[string]any{"type": "string", "enum": []string{"image", "video", "mask", "reference", "other"}}, + "tags": openAPIStringArray(), + }, + }, + "WebhookPayload": map[string]any{ + "type": "object", + "required": []string{"jobId", "status", "capability", "outputAssetIds", "updatedAt"}, + "properties": map[string]any{ + "jobId": map[string]any{"type": "string", "example": "job_mpqe3wtt_12ed738079"}, + "status": openAPIRef("GenerationStatus"), + "capability": openAPIRef("GenerationCapability"), + "outputAssetIds": openAPIStringArray(), + "error": openAPIJobError(), + "updatedAt": map[string]any{"type": "string", "format": "date-time"}, + }, + }, + } +} + +func openAPIPaths() map[string]any { + return map[string]any{ + "/api/v1/capabilities": map[string]any{ + "get": map[string]any{ + "summary": "List generation capabilities", + "responses": map[string]any{"200": openAPIJSONResponse("Capabilities and active providers", nil)}, + }, + }, + "/api/v1/assets": map[string]any{ + "get": map[string]any{ + "summary": "List assets visible to the authenticated API client", + "responses": map[string]any{"200": openAPIJSONResponse("Assets", map[string]any{ + "type": "object", "properties": map[string]any{"assets": map[string]any{"type": "array", "items": openAPIRef("Asset")}}, + })}, + }, + "post": map[string]any{ + "summary": "Upload files or register an external asset URL", + "requestBody": map[string]any{ + "required": true, + "content": map[string]any{ + "application/json": map[string]any{"schema": openAPIRef("RegisterAssetRequest")}, + "multipart/form-data": map[string]any{"schema": map[string]any{ + "type": "object", "properties": map[string]any{"files": map[string]any{"type": "array", "items": map[string]any{"type": "string", "format": "binary"}}}, + }}, + }, + }, + "responses": map[string]any{ + "201": openAPIJSONResponse("Created asset", nil), + "400": openAPIErrorResponse(), "401": openAPIErrorResponse(), + }, + }, + }, + "/api/v1/assets/{id}": map[string]any{ + "get": map[string]any{ + "summary": "Get one asset visible to the authenticated API client", + "parameters": []any{openAPIPathID()}, + "responses": map[string]any{ + "200": openAPIJSONResponse("Asset", map[string]any{"type": "object", "properties": map[string]any{"asset": openAPIRef("Asset")}}), + "404": openAPIErrorResponse(), + }, + }, + }, + "/api/v1/assets/{id}/download": map[string]any{ + "get": map[string]any{ + "summary": "Download an output or uploaded asset", + "parameters": []any{openAPIPathID()}, + "responses": map[string]any{ + "200": map[string]any{ + "description": "Binary file", + "headers": map[string]any{ + "Content-Disposition": map[string]any{"schema": map[string]any{"type": "string"}}, + "Content-Length": map[string]any{"schema": map[string]any{"type": "string"}}, + }, + "content": map[string]any{ + "application/octet-stream": openAPIBinaryContent(), + "image/png": openAPIBinaryContent(), + "image/jpeg": openAPIBinaryContent(), + "video/mp4": openAPIBinaryContent(), + }, + }, + "404": openAPIErrorResponse(), + }, + }, + }, + "/api/v1/jobs": map[string]any{ + "get": map[string]any{ + "summary": "List jobs for the authenticated API client", + "parameters": []any{ + openAPIQueryParameter("status", openAPIRef("GenerationStatus")), + openAPIQueryParameter("capability", openAPIRef("GenerationCapability")), + openAPIQueryParameter("limit", map[string]any{"type": "integer", "minimum": 1, "maximum": 200}), + openAPIQueryParameter("before", map[string]any{"type": "string", "format": "date-time"}), + }, + "responses": map[string]any{"200": openAPIJSONResponse("Jobs", map[string]any{ + "type": "object", "properties": map[string]any{"jobs": map[string]any{"type": "array", "items": openAPIRef("GenerationJob")}}, + })}, + }, + "post": map[string]any{ + "summary": "Create a queued generation job", + "parameters": []any{map[string]any{ + "name": "Idempotency-Key", "in": "header", "required": false, + "schema": map[string]any{"type": "string"}, + "description": "Reuse the same key for safe retries with the same request body.", + }}, + "requestBody": map[string]any{ + "required": true, + "content": map[string]any{"application/json": map[string]any{ + "schema": openAPIRef("CreateJobRequest"), + "examples": map[string]any{ + "imageGenerate": map[string]any{"summary": "Image generation", "value": map[string]any{ + "capability": "image.generate", "prompt": "生成一张专业产品主图", "width": 1440, "height": 2560, + "webhookUrl": "https://example.com/zhinian/webhook", + }}, + "videoGenerate": map[string]any{"summary": "Video generation", "value": map[string]any{ + "capability": "video.generate", "prompt": "生成一条 9:16 品牌短视频", + "settings": map[string]any{"ratio": "9:16", "duration": 5, "resolution": "720p"}, + }}, + }, + }}, + }, + "responses": map[string]any{ + "202": openAPIJSONResponse("Queued job", map[string]any{ + "type": "object", "properties": map[string]any{"job": openAPIRef("GenerationJob"), "reused": map[string]any{"type": "boolean"}}, + }), + "409": openAPIErrorResponse(), + }, + }, + }, + "/api/v1/jobs/{id}": map[string]any{ + "get": map[string]any{ + "summary": "Get one job", "parameters": []any{openAPIPathID()}, + "responses": map[string]any{ + "200": openAPIJSONResponse("Job", map[string]any{"type": "object", "properties": map[string]any{"job": openAPIRef("GenerationJob")}}), + "404": openAPIErrorResponse(), + }, + }, + }, + "/api/v1/jobs/{id}/cancel": map[string]any{ + "post": map[string]any{ + "summary": "Cancel a queued or running job", "parameters": []any{openAPIPathID()}, + "responses": map[string]any{ + "200": openAPIJSONResponse("Cancelled job", map[string]any{"type": "object", "properties": map[string]any{"job": openAPIRef("GenerationJob")}}), + "404": openAPIErrorResponse(), + }, + }, + }, + } +} + +func openAPIRef(name string) map[string]any { + return map[string]any{"$ref": "#/components/schemas/" + name} +} + +func openAPIStringArray() map[string]any { + return map[string]any{"type": "array", "items": map[string]any{"type": "string"}} +} + +func openAPIJobError() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "code": map[string]any{"oneOf": []any{map[string]any{"type": "string"}, map[string]any{"type": "number"}}}, + "message": map[string]any{"type": "string"}, + "retryable": map[string]any{"type": "boolean"}, + }, + } +} + +func openAPIPathID() map[string]any { + return map[string]any{"name": "id", "in": "path", "required": true, "schema": map[string]any{"type": "string"}} +} + +func openAPIQueryParameter(name string, schema map[string]any) map[string]any { + return map[string]any{"name": name, "in": "query", "required": false, "schema": schema} +} + +func openAPIJSONResponse(description string, schema map[string]any) map[string]any { + if schema == nil { + schema = map[string]any{"type": "object", "additionalProperties": true} + } + return map[string]any{ + "description": description, + "content": map[string]any{"application/json": map[string]any{"schema": schema}}, + } +} + +func openAPIErrorResponse() map[string]any { + return openAPIJSONResponse("Error", openAPIRef("ErrorResponse")) +} + +func openAPIBinaryContent() map[string]any { + return map[string]any{"schema": map[string]any{"type": "string", "format": "binary"}} +} diff --git a/backend/internal/httpapi/openapi_contract_test.go b/backend/internal/httpapi/openapi_contract_test.go new file mode 100644 index 0000000..1213886 --- /dev/null +++ b/backend/internal/httpapi/openapi_contract_test.go @@ -0,0 +1,67 @@ +package httpapi + +import ( + "reflect" + "sort" + "testing" +) + +func TestOpenAPIDocumentFreezesPublicSurface(t *testing.T) { + document := openAPIDocument("https://app.example.test") + if document["openapi"] != "3.1.0" { + t.Fatalf("openapi=%v", document["openapi"]) + } + + servers := document["servers"].([]any) + if got := servers[0].(map[string]any)["url"]; got != "https://app.example.test" { + t.Fatalf("server url=%v", got) + } + + components := document["components"].(map[string]any) + schemas := components["schemas"].(map[string]any) + gotSchemas := make([]string, 0, len(schemas)) + for name := range schemas { + gotSchemas = append(gotSchemas, name) + } + sort.Strings(gotSchemas) + wantSchemas := []string{ + "Asset", "CreateJobRequest", "ErrorResponse", "GenerationCapability", + "GenerationJob", "GenerationStatus", "PromptMaterial", "RegisterAssetRequest", + "WebhookPayload", + } + sort.Strings(wantSchemas) + if !reflect.DeepEqual(gotSchemas, wantSchemas) { + t.Fatalf("schemas=%v", gotSchemas) + } + + paths := document["paths"].(map[string]any) + gotPaths := make([]string, 0, len(paths)) + for path := range paths { + gotPaths = append(gotPaths, path) + } + sort.Strings(gotPaths) + wantPaths := []string{ + "/api/v1/assets", "/api/v1/assets/{id}", "/api/v1/assets/{id}/download", + "/api/v1/capabilities", "/api/v1/jobs", "/api/v1/jobs/{id}", + "/api/v1/jobs/{id}/cancel", + } + if !reflect.DeepEqual(gotPaths, wantPaths) { + t.Fatalf("paths=%v", gotPaths) + } + + jobs := paths["/api/v1/jobs"].(map[string]any) + if jobs["get"] == nil || jobs["post"] == nil { + t.Fatalf("jobs verbs=%v", jobs) + } + assets := paths["/api/v1/assets"].(map[string]any) + requestBody := assets["post"].(map[string]any)["requestBody"].(map[string]any) + content := requestBody["content"].(map[string]any) + if content["application/json"] == nil || content["multipart/form-data"] == nil { + t.Fatalf("asset content=%v", content) + } + createJob := schemas["CreateJobRequest"].(map[string]any) + properties := createJob["properties"].(map[string]any) + if properties["materials"] == nil || properties["settings"] == nil || properties["webhookUrl"] == nil { + t.Fatalf("create job properties=%v", properties) + } +} diff --git a/backend/internal/httpapi/platform_auth.go b/backend/internal/httpapi/platform_auth.go index 5a14bbb..7a7f59a 100644 --- a/backend/internal/httpapi/platform_auth.go +++ b/backend/internal/httpapi/platform_auth.go @@ -45,18 +45,34 @@ func (err *PlatformAuthError) Error() string { } type PlatformAuthorizer struct { - state AuthState - resolver SessionResolver - now func() time.Time + state AuthState + resolver SessionResolver + now func() time.Time + localDevelopmentFallback bool +} + +type PlatformAuthorizerOption func(*PlatformAuthorizer) + +// WithLocalDevelopmentFallback controls the privileged demo identity used by +// the explicit local backend. Production composition disables it even when a +// legacy auth environment value says authentication is optional. +func WithLocalDevelopmentFallback(enabled bool) PlatformAuthorizerOption { + return func(authorizer *PlatformAuthorizer) { authorizer.localDevelopmentFallback = enabled } } // NewPlatformAuthorizer creates the single HTTP-side platform authentication // seam shared by protected route Modules. -func NewPlatformAuthorizer(state AuthState, resolver SessionResolver) (*PlatformAuthorizer, error) { +func NewPlatformAuthorizer(state AuthState, resolver SessionResolver, options ...PlatformAuthorizerOption) (*PlatformAuthorizer, error) { if state.Configured && resolver == nil { return nil, fmt.Errorf("platform authorization: configured authentication requires a session resolver") } - return &PlatformAuthorizer{state: state, resolver: resolver, now: time.Now}, nil + authorizer := &PlatformAuthorizer{state: state, resolver: resolver, now: time.Now, localDevelopmentFallback: true} + for _, option := range options { + if option != nil { + option(authorizer) + } + } + return authorizer, nil } // Authorize returns either a database-refreshed platform Session, the exact @@ -83,9 +99,15 @@ func (authorizer *PlatformAuthorizer) Authorize(r *http.Request, requirement Pla } } - if !authorizer.state.Required { + if !authorizer.state.Required && authorizer.localDevelopmentFallback { return authorizePlatformRole(authorizer.localSession(), requirement) } + if !authorizer.state.Required { + return identity.Session{}, &PlatformAuthError{ + Kind: PlatformUnauthenticated, Status: http.StatusUnauthorized, + Message: "请先登录。", + } + } if !authorizer.state.Configured { return identity.Session{}, &PlatformAuthError{ Kind: PlatformConfigurationError, Status: http.StatusServiceUnavailable, diff --git a/backend/internal/httpapi/platform_auth_test.go b/backend/internal/httpapi/platform_auth_test.go index 20cf1b1..8664791 100644 --- a/backend/internal/httpapi/platform_auth_test.go +++ b/backend/internal/httpapi/platform_auth_test.go @@ -63,6 +63,22 @@ func TestPlatformAuthorizerUsesSharedChunkReader(t *testing.T) { } } +func TestPlatformAuthorizerCanDisableAnonymousAdministratorFallback(t *testing.T) { + authorizer, err := NewPlatformAuthorizer( + AuthState{Required: false, Configured: false}, + nil, + WithLocalDevelopmentFallback(false), + ) + if err != nil { + t.Fatal(err) + } + _, err = authorizer.Authorize(httptest.NewRequest(http.MethodGet, "/protected", nil), PlatformSuperAdmin) + var authErr *PlatformAuthError + if !errors.As(err, &authErr) || authErr.Status != http.StatusUnauthorized || authErr.Kind != PlatformUnauthenticated { + t.Fatalf("error=%#v, want unauthenticated/401", err) + } +} + type platformSessionResolverStub struct { outcome string session identity.Session diff --git a/backend/internal/httpapi/route_surface.go b/backend/internal/httpapi/route_surface.go new file mode 100644 index 0000000..28230e3 --- /dev/null +++ b/backend/internal/httpapi/route_surface.go @@ -0,0 +1,40 @@ +package httpapi + +import ( + "strings" +) + +// RouteSurface describes one externally visible route explicitly owned by the +// Go modular monolith. It is kept source-independent so contract tests can +// compare it with the checked-in Next.js surface before any cutover. +type RouteSurface struct { + Method string `json:"method"` + Path string `json:"path"` +} + +var goRouteSurface = []RouteSurface{ + {"DELETE", "/api/admin/accounts"}, {"GET", "/api/admin/accounts"}, {"PATCH", "/api/admin/accounts"}, {"POST", "/api/admin/accounts"}, {"PUT", "/api/admin/accounts"}, {"POST", "/api/admin/accounts/groups"}, {"POST", "/api/admin/accounts/password"}, {"GET", "/api/admin/billing"}, {"PATCH", "/api/admin/billing/account"}, {"POST", "/api/admin/billing/adjustments"}, {"GET", "/api/admin/billing/prices"}, {"PATCH", "/api/admin/billing/prices/{id}"}, {"DELETE", "/api/admin/organizations"}, {"GET", "/api/admin/organizations"}, {"PATCH", "/api/admin/organizations"}, {"POST", "/api/admin/organizations"}, {"GET", "/api/admin/usage"}, + {"GET", "/api/assets"}, {"POST", "/api/assets"}, {"POST", "/api/assets/upload"}, {"DELETE", "/api/assets/{id}"}, {"GET", "/api/assets/{id}/download"}, + {"GET", "/api/auth/callback"}, {"GET", "/api/auth/captcha"}, {"GET", "/api/auth/login"}, {"GET", "/api/auth/logout"}, {"POST", "/api/auth/logout"}, {"GET", "/api/auth/me"}, {"POST", "/api/auth/password"}, {"POST", "/api/auth/password/change"}, + {"GET", "/api/billing"}, {"POST", "/api/billing/quote"}, {"GET", "/api/generations/image"}, {"POST", "/api/generations/image"}, {"DELETE", "/api/generations/image/{id}"}, {"GET", "/api/generations/image/{id}"}, {"POST", "/api/generations/image/{id}/retry"}, {"GET", "/api/generations/video"}, {"POST", "/api/generations/video"}, {"DELETE", "/api/generations/video/{id}"}, {"GET", "/api/generations/video/{id}"}, + {"GET", "/api/health"}, {"GET", "/api/image-templates"}, {"POST", "/api/image-templates"}, {"DELETE", "/api/image-templates/{id}"}, {"PATCH", "/api/image-templates/{id}"}, {"POST", "/api/internal/worker/tick"}, {"DELETE", "/api/logs"}, {"GET", "/api/logs"}, {"POST", "/api/prompt/assemble"}, {"GET", "/api/ready"}, {"GET", "/api/settings"}, {"POST", "/api/settings"}, {"GET", "/api/usage"}, + {"GET", "/api/v1/assets"}, {"POST", "/api/v1/assets"}, {"GET", "/api/v1/assets/{id}"}, {"GET", "/api/v1/assets/{id}/download"}, {"GET", "/api/v1/capabilities"}, {"GET", "/api/v1/jobs"}, {"POST", "/api/v1/jobs"}, {"GET", "/api/v1/jobs/{id}"}, {"POST", "/api/v1/jobs/{id}/cancel"}, {"GET", "/api/v1/openapi.json"}, {"GET", "/generated-results/{path...}"}, {"GET", "/uploads/{path...}"}, +} + +func GoRouteSurface() []RouteSurface { return append([]RouteSurface(nil), goRouteSurface...) } + +func MatchSurfacePath(pattern, path string) bool { + if strings.HasSuffix(pattern, "/{path...}") { + prefix := strings.TrimSuffix(pattern, "{path...}") + return strings.HasPrefix(path, prefix) && strings.TrimPrefix(path, prefix) != "" + } + if strings.Contains(pattern, "{id}") { + prefix, suffix, _ := strings.Cut(pattern, "{id}") + if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) { + return false + } + value := strings.TrimSuffix(strings.TrimPrefix(path, prefix), suffix) + return value != "" && !strings.Contains(value, "/") + } + return pattern == path +} diff --git a/backend/internal/httpapi/route_surface_test.go b/backend/internal/httpapi/route_surface_test.go new file mode 100644 index 0000000..349a141 --- /dev/null +++ b/backend/internal/httpapi/route_surface_test.go @@ -0,0 +1,53 @@ +package httpapi + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "runtime" + "sort" + "testing" +) + +func TestGoRouteSurfaceExactlyMatchesCheckedInNextManifest(t *testing.T) { + _, source, _, _ := runtime.Caller(0) + raw, err := os.ReadFile(filepath.Join(filepath.Dir(source), "../../../contracts/http/route-surface.v1.json")) + if err != nil { + t.Fatal(err) + } + var fixture struct { + Version int `json:"version"` + Routes []RouteSurface `json:"routes"` + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + if fixture.Version != 1 { + t.Fatalf("version=%d", fixture.Version) + } + got := GoRouteSurface() + less := func(items []RouteSurface) { + sort.Slice(items, func(i, j int) bool { + if items[i].Method != items[j].Method { + return items[i].Method < items[j].Method + } + return items[i].Path < items[j].Path + }) + } + less(got) + less(fixture.Routes) + if !reflect.DeepEqual(got, fixture.Routes) { + t.Fatalf("Go route surface drift\ngot=%#v\nwant=%#v", got, fixture.Routes) + } +} +func TestMatchSurfacePath(t *testing.T) { + for _, test := range []struct { + pattern, path string + want bool + }{{"/api/x/{id}", "/api/x/a", true}, {"/api/x/{id}", "/api/x/a/b", false}, {"/uploads/{path...}", "/uploads/a/b", true}, {"/uploads/{path...}", "/uploads/", false}, {"/generated-results/{path...}", "/generated-results/", false}, {"/api/health", "/api/health", true}} { + if got := MatchSurfacePath(test.pattern, test.path); got != test.want { + t.Fatalf("%q %q=%v", test.pattern, test.path, got) + } + } +} diff --git a/backend/internal/httpapi/usage.go b/backend/internal/httpapi/usage.go new file mode 100644 index 0000000..11ab97e --- /dev/null +++ b/backend/internal/httpapi/usage.go @@ -0,0 +1,117 @@ +package httpapi + +import ( + "errors" + "net/http" + "strings" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage" +) + +type UsageReporter = usage.Reporter +type usageHandler struct { + authorizer *PlatformAuthorizer + reporter UsageReporter + now func() time.Time +} + +func NewUsageHandler(authorizer *PlatformAuthorizer, reporter UsageReporter, now func() time.Time) http.Handler { + if now == nil { + now = time.Now + } + return &usageHandler{authorizer: authorizer, reporter: reporter, now: now} +} +func (h *usageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/usage" && r.URL.Path != "/api/admin/usage" { + http.NotFound(w, r) + return + } + if !allow(w, r, http.MethodGet) { + return + } + if h.authorizer == nil || h.reporter == nil { + writeAPIError(w, 500, "服务器内部错误。") + return + } + if r.URL.Path == "/api/usage" { + h.personal(w, r) + return + } + h.admin(w, r) +} +func (h *usageHandler) personal(w http.ResponseWriter, r *http.Request) { + session, err := h.authorizer.Authorize(r, PlatformApp) + if err != nil { + writeAuthError(w, err) + return + } + preset := usage.Preset(r.URL.Query().Get("preset")) + switch preset { + case usage.PresetToday, usage.Preset7Days, usage.Preset30Days, usage.PresetMonth: + default: + preset = usage.PresetMonth + } + report, err := h.reporter.Personal(r.Context(), usage.PersonalRequest{AccountID: session.User.ID, Preset: preset, Now: h.now()}) + if err != nil { + writeAPIError(w, 500, "服务器内部错误。") + return + } + writeJSON(w, 200, report) +} +func (h *usageHandler) admin(w http.ResponseWriter, r *http.Request) { + session, err := h.authorizer.Authorize(r, PlatformAdmin) + if err != nil { + writeAuthError(w, err) + return + } + query := r.URL.Query() + capability, provider := strings.TrimSpace(query.Get("capability")), strings.TrimSpace(query.Get("provider")) + if capability != "" && capability != "image.generate" && capability != "video.generate" { + writeAPIError(w, 400, "不支持的功能类型。") + return + } + if provider != "" && provider != "volcengine-visual" && provider != "evolink" && provider != "seedance" && provider != "bailian" { + writeAPIError(w, 400, "不支持的服务商。") + return + } + startDate, endDate := strings.TrimSpace(query.Get("startDate")), strings.TrimSpace(query.Get("endDate")) + if !validDate(startDate) || !validDate(endDate) { + writeAPIError(w, 400, "日期格式无效。") + return + } + request := usage.AdminRequest{Requester: usage.Requester{AccountID: session.User.ID, OrganizationID: session.User.OrganizationID, Role: session.User.Role}, StartDate: startDate, EndDate: endDate, Capability: capability, Provider: provider, Now: h.now()} + if session.User.Role == "super_admin" { + request.OrganizationID = strings.TrimSpace(query.Get("organizationId")) + request.OwnerID = strings.TrimSpace(query.Get("ownerId")) + } else { + request.OrganizationID = session.User.OrganizationID + request.RedactAccounts = true + } + report, err := h.reporter.Admin(r.Context(), request) + if err != nil { + if errors.Is(err, usage.ErrDateRangeTooLong) { + writeAPIError(w, 400, "单次查询最多支持 10 年。") + return + } + if errors.Is(err, usage.ErrInvalidDateRange) { + writeAPIError(w, 400, "开始日期不能晚于结束日期。") + return + } + writeAPIError(w, 500, "服务器内部错误。") + return + } + if request.RedactAccounts { + report.Accounts = []any{} + report.Recent = []any{} + report.Options.Accounts = []usage.Option{} + } + writeJSON(w, 200, report) +} +func validDate(value string) bool { + if value == "" { + return true + } + _, err := time.Parse("2006-01-02", value) + return err == nil +} diff --git a/backend/internal/httpapi/usage_test.go b/backend/internal/httpapi/usage_test.go new file mode 100644 index 0000000..cf7b997 --- /dev/null +++ b/backend/internal/httpapi/usage_test.go @@ -0,0 +1,94 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage" +) + +func TestUsageRoutesEnforcePersonalAndAdminScope(t *testing.T) { + reporter := &usageReporterStub{} + user := usageTestHandler(t, identity.Session{AuthMode: identity.AuthModeUser, User: identity.User{ID: "db-user", ClientID: "platform", OrganizationID: "db-org", Role: "user"}}, reporter) + if got := serveJSON(t, user, http.MethodGet, "/api/usage?preset=7d", nil); got.Code != 200 || reporter.personal.AccountID != "db-user" || reporter.personal.Preset != usage.Preset7Days { + t.Fatalf("status=%d request=%+v", got.Code, reporter.personal) + } + if got := serveJSON(t, user, http.MethodGet, "/api/admin/usage", nil); got.Code != 403 { + t.Fatalf("user admin status=%d", got.Code) + } + + orgAdmin := usageTestHandler(t, identity.Session{AuthMode: identity.AuthModeAdmin, User: identity.User{ID: "admin", ClientID: "platform", OrganizationID: "db-org", Role: "organization_admin"}}, reporter) + got := serveJSON(t, orgAdmin, http.MethodGet, "/api/admin/usage?organizationId=attacker&ownerId=other&provider=bailian", nil) + if got.Code != 200 || reporter.admin.OrganizationID != "db-org" || reporter.admin.OwnerID != "" || reporter.admin.Provider != "bailian" || !reporter.admin.RedactAccounts { + t.Fatalf("status=%d request=%+v body=%s", got.Code, reporter.admin, got.Body.String()) + } + + super := usageTestHandler(t, identity.Session{AuthMode: identity.AuthModeAdmin, User: identity.User{ID: "root", ClientID: "platform", Role: "super_admin"}}, reporter) + got = serveJSON(t, super, http.MethodGet, "/api/admin/usage?organizationId=org-x&ownerId=user-x&capability=image.generate", nil) + if got.Code != 200 || reporter.admin.OrganizationID != "org-x" || reporter.admin.OwnerID != "user-x" || reporter.admin.RedactAccounts { + t.Fatalf("status=%d request=%+v", got.Code, reporter.admin) + } +} + +func TestUsageValidatesFiltersAndHidesInfrastructureErrors(t *testing.T) { + reporter := &usageReporterStub{} + h := usageTestHandler(t, identity.Session{AuthMode: identity.AuthModeAdmin, User: identity.User{ID: "root", ClientID: "platform", Role: "super_admin"}}, reporter) + for _, path := range []string{"/api/admin/usage?capability=bad", "/api/admin/usage?provider=mock", "/api/admin/usage?startDate=not-date"} { + if got := serveJSON(t, h, http.MethodGet, path, nil); got.Code != 400 { + t.Fatalf("%s status=%d", path, got.Code) + } + } + reporter.err = context.DeadlineExceeded + if got := serveJSON(t, h, http.MethodGet, "/api/admin/usage", nil); got.Code != 500 || got.Body.String() != "{\"error\":\"服务器内部错误。\"}\n" { + t.Fatalf("status/body=%d %q", got.Code, got.Body.String()) + } +} + +func TestUsageMapsDomainDateRangeErrorsToBadRequest(t *testing.T) { + reporter := &usageReporterStub{} + h := usageTestHandler(t, identity.Session{AuthMode: identity.AuthModeAdmin, User: identity.User{ID: "root", ClientID: "platform", Role: "super_admin"}}, reporter) + + reporter.err = usage.ErrDateRangeTooLong + if got := serveJSON(t, h, http.MethodGet, "/api/admin/usage?startDate=2010-01-01&endDate=2026-01-01", nil); got.Code != 400 || got.Body.String() != "{\"error\":\"单次查询最多支持 10 年。\"}\n" { + t.Fatalf("long range status/body=%d %q", got.Code, got.Body.String()) + } + + reporter.err = fmt.Errorf("wrapped: %w", usage.ErrInvalidDateRange) + if got := serveJSON(t, h, http.MethodGet, "/api/admin/usage?startDate=2026-08-13&endDate=2026-08-12", nil); got.Code != 400 || got.Body.String() != "{\"error\":\"开始日期不能晚于结束日期。\"}\n" { + t.Fatalf("reverse range status/body=%d %q", got.Code, got.Body.String()) + } + + reporter.err = errors.New("database unavailable") + if got := serveJSON(t, h, http.MethodGet, "/api/admin/usage", nil); got.Code != 500 { + t.Fatalf("infrastructure status=%d", got.Code) + } +} + +type usageReporterStub struct { + personal usage.PersonalRequest + admin usage.AdminRequest + err error +} + +func (s *usageReporterStub) Personal(_ context.Context, request usage.PersonalRequest) (usage.PersonalReport, error) { + s.personal = request + return usage.PersonalReport{Preset: request.Preset}, s.err +} +func (s *usageReporterStub) Admin(_ context.Context, request usage.AdminRequest) (usage.AdminReport, error) { + s.admin = request + return usage.AdminReport{}, s.err +} + +func usageTestHandler(t *testing.T, session identity.Session, reporter UsageReporter) http.Handler { + t.Helper() + authorizer, err := NewPlatformAuthorizer(AuthState{Required: true, Configured: true}, &fixedSessionResolver{session: session}) + if err != nil { + t.Fatal(err) + } + return NewUsageHandler(authorizer, reporter, func() time.Time { return time.Date(2026, 8, 13, 0, 0, 0, 0, time.UTC) }) +} diff --git a/backend/internal/identity/password_change.go b/backend/internal/identity/password_change.go new file mode 100644 index 0000000..32c9a5a --- /dev/null +++ b/backend/internal/identity/password_change.go @@ -0,0 +1,102 @@ +package identity + +import ( + "context" + "errors" + "fmt" + "strings" + "time" +) + +type PasswordChanger interface { + ChangeOwnPassword(context.Context, string, string, string, time.Time) (AuthorizationSnapshot, error) +} + +type PasswordChangeCommand struct { + AccountID, CurrentPassword, NewPassword string +} + +type PasswordChangeFailure string + +const ( + PasswordChangeInvalidInput PasswordChangeFailure = "invalid_input" + PasswordChangeInvalidNewPassword PasswordChangeFailure = "invalid_new_password" + PasswordChangeNotFound PasswordChangeFailure = "not_found" + PasswordChangeCurrentIncorrect PasswordChangeFailure = "current_password_incorrect" +) + +var ErrPasswordChange = errors.New("password change failed") + +type PasswordChangeError struct{ Reason PasswordChangeFailure } + +func (e *PasswordChangeError) Error() string { + return fmt.Sprintf("%s: %s", ErrPasswordChange, e.Reason) +} +func (e *PasswordChangeError) Unwrap() error { return ErrPasswordChange } +func IsPasswordChangeFailure(err error, reason PasswordChangeFailure) bool { + var target *PasswordChangeError + return errors.As(err, &target) && target.Reason == reason +} + +type PasswordChange struct { + store PasswordChanger + now func() time.Time +} + +func NewPasswordChange(store PasswordChanger, now func() time.Time) *PasswordChange { + if now == nil { + now = time.Now + } + return &PasswordChange{store: store, now: now} +} + +func (change *PasswordChange) Change(ctx context.Context, command PasswordChangeCommand) (Session, error) { + command.AccountID = strings.TrimSpace(command.AccountID) + command.CurrentPassword = strings.TrimSpace(command.CurrentPassword) + command.NewPassword = strings.TrimSpace(command.NewPassword) + if command.AccountID == "" || command.CurrentPassword == "" { + return Session{}, &PasswordChangeError{Reason: PasswordChangeInvalidInput} + } + if len(command.NewPassword) < 8 { + return Session{}, &PasswordChangeError{Reason: PasswordChangeInvalidNewPassword} + } + if change == nil || change.store == nil { + return Session{}, fmt.Errorf("password change is not configured") + } + now := change.now() + snapshot, err := change.store.ChangeOwnPassword(ctx, command.AccountID, command.CurrentPassword, command.NewPassword, now) + if err != nil { + return Session{}, err + } + return sessionFromSnapshot(snapshot, now) +} + +func sessionFromSnapshot(snapshot AuthorizationSnapshot, now time.Time) (Session, error) { + account := snapshot.Account + if account.Status != "active" { + return Session{}, &PasswordChangeError{Reason: PasswordChangeNotFound} + } + authMode, authorities, valid := roleClaims(account.Role) + if !valid { + return Session{}, NewPasswordLoginError(LoginFailureInvalidRole) + } + organizationName := "" + if account.Role != "super_admin" { + if account.OrganizationID == "" { + return Session{}, NewPasswordLoginError(LoginFailureOrganizationRequired) + } + if snapshot.Organization == nil || snapshot.Organization.ID != account.OrganizationID || snapshot.Organization.Status != "active" { + return Session{}, NewPasswordLoginError(LoginFailureOrganizationNotActive) + } + organizationName = snapshot.Organization.Name + } else if snapshot.Organization != nil && snapshot.Organization.ID == account.OrganizationID { + organizationName = snapshot.Organization.Name + } + version := account.SessionVersion + return Session{Version: 1, AuthMode: authMode, IssuedAt: now.Unix(), ExpiresAt: now.Add(passwordSessionTTL).Unix(), SessionVersion: &version, User: User{ + ID: account.ID, Subject: account.ID, Username: account.Phone, Phone: account.Phone, + DisplayName: account.DisplayName, ClientID: "platform", OrganizationID: account.OrganizationID, + OrganizationName: organizationName, Role: account.Role, Status: account.Status, + Authorities: authorities, Scope: []string{}, + }}, nil +} diff --git a/backend/internal/identity/password_change_test.go b/backend/internal/identity/password_change_test.go new file mode 100644 index 0000000..e35d875 --- /dev/null +++ b/backend/internal/identity/password_change_test.go @@ -0,0 +1,44 @@ +package identity + +import ( + "context" + "testing" + "time" +) + +type passwordChangeStoreStub struct { + snapshot AuthorizationSnapshot + err error + id, current, next string +} + +func (s *passwordChangeStoreStub) ChangeOwnPassword(_ context.Context, id, current, next string, _ time.Time) (AuthorizationSnapshot, error) { + s.id, s.current, s.next = id, current, next + return s.snapshot, s.err +} + +func TestPasswordChangeReturnsRefreshedSession(t *testing.T) { + now := time.Unix(1770000000, 0) + store := &passwordChangeStoreStub{snapshot: AuthorizationSnapshot{ + Account: AccountSnapshot{ID: "user-1", Phone: "13800138000", DisplayName: "User", Role: "user", OrganizationID: "org-1", Status: "active", SessionVersion: 8}, + Organization: &OrganizationSnapshot{ID: "org-1", Name: "Acme", Status: "active"}, + }} + session, err := NewPasswordChange(store, func() time.Time { return now }).Change(context.Background(), PasswordChangeCommand{AccountID: "user-1", CurrentPassword: " current-pass ", NewPassword: "next-pass"}) + if err != nil { + t.Fatal(err) + } + if store.current != "current-pass" || store.next != "next-pass" || session.SessionVersion == nil || *session.SessionVersion != 8 || session.User.OrganizationName != "Acme" { + t.Fatalf("store=%+v session=%+v", store, session) + } +} + +func TestPasswordChangeValidatesInput(t *testing.T) { + store := &passwordChangeStoreStub{} + _, err := NewPasswordChange(store, nil).Change(context.Background(), PasswordChangeCommand{AccountID: "user-1", CurrentPassword: "old", NewPassword: "short"}) + if !IsPasswordChangeFailure(err, PasswordChangeInvalidNewPassword) { + t.Fatalf("err=%v", err) + } + if store.id != "" { + t.Fatal("store called for invalid input") + } +} diff --git a/backend/internal/jobs/job.go b/backend/internal/jobs/job.go index 6dcb33b..194f613 100644 --- a/backend/internal/jobs/job.go +++ b/backend/internal/jobs/job.go @@ -33,39 +33,42 @@ func (status Status) Valid() bool { } type Job struct { - ID string `json:"id"` - OwnerID string `json:"ownerId"` - ExternalClientID string `json:"externalClientId,omitempty"` - Capability string `json:"capability"` - Provider string `json:"provider"` - ReqKey string `json:"reqKey"` - Status Status `json:"status"` - Prompt string `json:"prompt,omitempty"` - InputAssetIDs []string `json:"inputAssetIds"` - InputURLs []string `json:"inputUrls"` - OutputAssetIDs []string `json:"outputAssetIds"` - ProviderTaskID string `json:"providerTaskId,omitempty"` - RequestPayload json.RawMessage `json:"requestPayload"` - ResponsePayload json.RawMessage `json:"responsePayload,omitempty"` - Error *JobError `json:"error,omitempty"` - RetryOf string `json:"retryOf,omitempty"` - IdempotencyKey string `json:"idempotencyKey,omitempty"` - IdempotencyFingerprint string `json:"idempotencyFingerprint,omitempty"` - Priority int `json:"priority,omitempty"` - Attempts int `json:"attempts,omitempty"` - MaxAttempts int `json:"maxAttempts,omitempty"` - ScheduledAt time.Time `json:"scheduledAt,omitempty"` - LockedAt *time.Time `json:"lockedAt,omitempty"` - LockedBy string `json:"lockedBy,omitempty"` - StartedAt *time.Time `json:"startedAt,omitempty"` - CompletedAt *time.Time `json:"completedAt,omitempty"` - WebhookURL string `json:"webhookUrl,omitempty"` - WebhookAttempts int `json:"webhookAttempts,omitempty"` - WebhookLastStatus json.RawMessage `json:"webhookLastStatus,omitempty"` - UsageContext json.RawMessage `json:"usageContext,omitempty"` - Billing json.RawMessage `json:"billing,omitempty"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + OwnerID string `json:"ownerId"` + ExternalClientID string `json:"externalClientId,omitempty"` + Capability string `json:"capability"` + Provider string `json:"provider"` + ReqKey string `json:"reqKey"` + Status Status `json:"status"` + Prompt string `json:"prompt,omitempty"` + InputAssetIDs []string `json:"inputAssetIds"` + InputURLs []string `json:"inputUrls"` + OutputAssetIDs []string `json:"outputAssetIds"` + ProviderTaskID string `json:"providerTaskId,omitempty"` + ProviderDispatchStartedAt *time.Time `json:"providerDispatchStartedAt,omitempty"` + RequestPayload json.RawMessage `json:"requestPayload"` + ResponsePayload json.RawMessage `json:"responsePayload,omitempty"` + Error *JobError `json:"error,omitempty"` + RetryOf string `json:"retryOf,omitempty"` + IdempotencyKey string `json:"idempotencyKey,omitempty"` + IdempotencyFingerprint string `json:"idempotencyFingerprint,omitempty"` + Priority int `json:"priority,omitempty"` + Attempts int `json:"attempts,omitempty"` + MaxAttempts int `json:"maxAttempts,omitempty"` + ScheduledAt time.Time `json:"scheduledAt,omitempty"` + LockedAt *time.Time `json:"lockedAt,omitempty"` + LockedBy string `json:"lockedBy,omitempty"` + StartedAt *time.Time `json:"startedAt,omitempty"` + CompletedAt *time.Time `json:"completedAt,omitempty"` + DispatchReadyAt *time.Time `json:"dispatchReadyAt,omitempty"` + FinalizedAt *time.Time `json:"finalizedAt,omitempty"` + WebhookURL string `json:"webhookUrl,omitempty"` + WebhookAttempts int `json:"webhookAttempts,omitempty"` + WebhookLastStatus json.RawMessage `json:"webhookLastStatus,omitempty"` + UsageContext json.RawMessage `json:"usageContext,omitempty"` + Billing json.RawMessage `json:"billing,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } type JobError struct { @@ -103,6 +106,7 @@ type Error struct { func (err *Error) Error() string { return err.Message } var ErrUniqueIdempotency = errors.New("generation job idempotency key already exists") +var ErrTransitionConflict = errors.New("generation job transition conflict") type ListFilter struct { Scope Scope @@ -115,6 +119,9 @@ type ListFilter struct { type CreateCommand struct { Job Job IdempotencyBody map[string]any + // HoldDispatch keeps a newly persisted job invisible to workers until an + // external prerequisite (currently the idempotent wallet charge) is durable. + HoldDispatch bool } func NormalizePriority(value int) int { diff --git a/backend/internal/jobs/jobs_test.go b/backend/internal/jobs/jobs_test.go index f2ee4ad..a3e7746 100644 --- a/backend/internal/jobs/jobs_test.go +++ b/backend/internal/jobs/jobs_test.go @@ -117,6 +117,9 @@ func TestWorkerSchedulesRetryReleasesRunningAndSettlesTerminal(t *testing.T) { if !test.scheduled.IsZero() && !stored.ScheduledAt.Equal(test.scheduled) { t.Fatalf("scheduled = %s, want %s", stored.ScheduledAt, test.scheduled) } + if test.want == "retry_scheduled" && stored.Error != nil { + t.Fatalf("retry must clear the transient error, got %#v", stored.Error) + } if test.want == "processed" && refunds.calls != 1 { t.Fatalf("refund calls = %d, want 1", refunds.calls) } @@ -124,6 +127,78 @@ func TestWorkerSchedulesRetryReleasesRunningAndSettlesTerminal(t *testing.T) { } } +func TestWorkerRecoversTerminalProviderResultWithoutResubmittingAndFinalizesAfterEffects(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + locked := now.Add(-time.Minute) + job := Job{ID: "terminal", OwnerID: "owner", Capability: "image.generate", Provider: "fixture", Status: StatusSucceeded, LockedAt: &locked, LockedBy: "worker-1"} + store := newMemoryJobStore() + store.claimed = []Job{job} + store.jobs[job.ID] = job + processor := &processorStub{job: job} + usage := &usageStub{} + webhooks := &webhookStub{result: WebhookResult{Attempts: 1, LastStatus: map[string]any{"ok": true}}} + worker := NewWorker(store, processor, nil, usage, webhooks, WorkerConfig{}, func() time.Time { return now }) + + result, err := worker.Tick(context.Background(), "worker-1") + if err != nil || len(result.Jobs) != 1 || result.Jobs[0].Action != "processed" { + t.Fatalf("Tick = %#v, %v", result, err) + } + stored := store.jobs[job.ID] + if processor.calls != 1 || usage.calls != 1 || webhooks.calls != 1 || stored.FinalizedAt == nil || stored.LockedBy != "" { + t.Fatalf("processor=%d usage=%d webhook=%d stored=%#v", processor.calls, usage.calls, webhooks.calls, stored) + } +} + +func TestWorkerLostLeaseCannotOverwriteCancellation(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + locked := now.Add(-time.Minute) + claimed := Job{ID: "race", OwnerID: "owner", Capability: "image.generate", Provider: "fixture", Status: StatusRunning, LockedAt: &locked, LockedBy: "worker-1"} + store := newMemoryJobStore() + store.claimed = []Job{claimed} + cancelled := claimed + cancelled.Status, cancelled.LockedAt, cancelled.LockedBy = StatusCancelled, nil, "" + store.jobs[claimed.ID] = cancelled + worker := NewWorker(store, &processorStub{job: Job{ID: claimed.ID, Status: StatusSucceeded}}, nil, nil, nil, WorkerConfig{}, func() time.Time { return now }) + + _, err := worker.Tick(context.Background(), "worker-1") + if !errors.Is(err, ErrTransitionConflict) { + t.Fatalf("Tick error = %v, want transition conflict", err) + } + if got := store.jobs[claimed.ID]; got.Status != StatusCancelled { + t.Fatalf("stored = %#v", got) + } +} + +func TestWorkerCountsEachFailedAdvanceExactlyOnce(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + store := newMemoryJobStore() + job := Job{ID: "retry", OwnerID: "owner", Capability: "image.generate", Provider: "fixture", Status: StatusRunning, LockedBy: "worker-1", MaxAttempts: 3} + store.claimed, store.jobs[job.ID] = []Job{job}, job + worker := NewWorker(store, &processorStub{err: errors.New("temporary")}, nil, nil, nil, WorkerConfig{RetryBase: time.Second}, func() time.Time { return now }) + + if _, err := worker.Tick(context.Background(), "worker-1"); err != nil { + t.Fatal(err) + } + got := store.jobs[job.ID] + if got.Attempts != 1 || got.Status != StatusQueued { + t.Fatalf("after one failed advance = %#v", got) + } +} + +func TestWorkerRetryPreservesKnownProviderTaskToAvoidDuplicateSubmission(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + store := newMemoryJobStore() + job := Job{ID: "poll", OwnerID: "owner", Capability: "image.generate", Provider: "fixture", ProviderTaskID: "provider-task", Status: StatusRunning, LockedBy: "worker-1", MaxAttempts: 3} + store.claimed, store.jobs[job.ID] = []Job{job}, job + worker := NewWorker(store, &processorStub{err: errors.New("query timeout")}, nil, nil, nil, WorkerConfig{}, func() time.Time { return now }) + if _, err := worker.Tick(context.Background(), "worker-1"); err != nil { + t.Fatal(err) + } + if got := store.jobs[job.ID]; got.ProviderTaskID != "provider-task" || got.Status != StatusQueued { + t.Fatalf("retry lost provider recovery handle: %#v", got) + } +} + func fixtureCreateCommand(prompt string) CreateCommand { body := map[string]any{ "capability": "image.generate", "prompt": prompt, @@ -143,6 +218,11 @@ type memoryJobStore struct { race *Job } +func (store *memoryJobStore) DeleteJob(_ context.Context, id string) error { + delete(store.jobs, id) + return nil +} + func newMemoryJobStore() *memoryJobStore { return &memoryJobStore{jobs: map[string]Job{}} } func (store *memoryJobStore) ListJobs(context.Context, ListFilter) ([]Job, error) { return nil, nil } func (store *memoryJobStore) FindJob(_ context.Context, scope Scope, id string) (Job, bool, error) { @@ -168,12 +248,27 @@ func (store *memoryJobStore) CreateJob(_ context.Context, job Job) (Job, error) } func (store *memoryJobStore) UpdateJob(_ context.Context, id string, patch Patch) (Job, error) { job := store.jobs[id] + if len(patch.ExpectedStatuses) != 0 { + matched := false + for _, status := range patch.ExpectedStatuses { + matched = matched || job.Status == status + } + if !matched { + return Job{}, ErrTransitionConflict + } + } + if patch.ExpectedLockedBy != nil && job.LockedBy != *patch.ExpectedLockedBy { + return Job{}, ErrTransitionConflict + } if patch.Status != nil { job.Status = *patch.Status } if patch.Error != nil { job.Error = patch.Error } + if patch.ClearError { + job.Error = nil + } if patch.Attempts != nil { job.Attempts = *patch.Attempts } @@ -190,12 +285,28 @@ func (store *memoryJobStore) UpdateJob(_ context.Context, id string, patch Patch if patch.ClearProviderTaskID { job.ProviderTaskID = "" } + if patch.ProviderTaskID != nil { + job.ProviderTaskID = *patch.ProviderTaskID + } + if patch.ProviderDispatchStartedAt != nil { + value := *patch.ProviderDispatchStartedAt + job.ProviderDispatchStartedAt = &value + } + if patch.ClearProviderDispatch { + job.ProviderDispatchStartedAt = nil + } + if patch.SetResponsePayload { + job.ResponsePayload = append([]byte(nil), patch.ResponsePayload...) + } if patch.WebhookAttempts != nil { job.WebhookAttempts = *patch.WebhookAttempts } if patch.SetWebhookStatus { job.WebhookLastStatus = patch.WebhookLastStatus } + if patch.FinalizedAt != nil { + job.FinalizedAt = patch.FinalizedAt + } store.jobs[id] = job return job, nil } @@ -211,8 +322,29 @@ func (stub *refundStub) Refund(_ context.Context, job Job, _ string) (Job, error } type processorStub struct { - job Job - err error + job Job + err error + calls int } -func (stub *processorStub) Advance(context.Context, Job) (Job, error) { return stub.job, stub.err } +func (stub *processorStub) Advance(_ context.Context, input Job) (Job, error) { + stub.calls++ + if stub.job.ID == "" { + stub.job = input + } + return stub.job, stub.err +} + +type usageStub struct{ calls int } + +func (stub *usageStub) Record(context.Context, Job) error { stub.calls++; return nil } + +type webhookStub struct { + calls int + result WebhookResult +} + +func (stub *webhookStub) Deliver(context.Context, Job) (WebhookResult, error) { + stub.calls++ + return stub.result, nil +} diff --git a/backend/internal/jobs/loop.go b/backend/internal/jobs/loop.go new file mode 100644 index 0000000..d78be16 --- /dev/null +++ b/backend/internal/jobs/loop.go @@ -0,0 +1,80 @@ +package jobs + +import ( + "context" + "sync" + "sync/atomic" + "time" +) + +type TickRunner interface { + Tick(context.Context, string) (TickResult, error) +} +type LoopConfig struct { + Interval time.Duration + WorkerID string +} +type WorkerLoop struct { + runner TickRunner + config LoopConfig + running atomic.Bool + mu sync.Mutex + cancel context.CancelFunc + done chan struct{} +} + +func NewWorkerLoop(r TickRunner, c LoopConfig) *WorkerLoop { + if c.Interval <= 0 { + c.Interval = 5 * time.Second + } + if c.WorkerID == "" { + c.WorkerID = "embedded-worker" + } + return &WorkerLoop{runner: r, config: c} +} +func (l *WorkerLoop) Tick(ctx context.Context) (TickResult, bool) { + if l == nil || l.runner == nil || !l.running.CompareAndSwap(false, true) { + return TickResult{}, false + } + defer l.running.Store(false) + r, e := l.runner.Tick(ctx, l.config.WorkerID) + if e != nil { + return TickResult{WorkerID: l.config.WorkerID, Jobs: []TickJob{{Action: "failed", Error: e.Error()}}}, true + } + return r, true +} +func (l *WorkerLoop) Start(parent context.Context) { + l.mu.Lock() + defer l.mu.Unlock() + if l.cancel != nil { + return + } + ctx, cancel := context.WithCancel(parent) + l.cancel = cancel + l.done = make(chan struct{}) + go func() { + defer close(l.done) + ticker := time.NewTicker(l.config.Interval) + defer ticker.Stop() + for { + _, _ = l.Tick(ctx) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() +} +func (l *WorkerLoop) Stop() { + l.mu.Lock() + cancel, done := l.cancel, l.done + l.cancel = nil + l.done = nil + l.mu.Unlock() + if cancel == nil { + return + } + cancel() + <-done +} diff --git a/backend/internal/jobs/loop_test.go b/backend/internal/jobs/loop_test.go new file mode 100644 index 0000000..a4c2bb0 --- /dev/null +++ b/backend/internal/jobs/loop_test.go @@ -0,0 +1,75 @@ +package jobs + +import ( + "context" + "sync" + "testing" + "time" +) + +type tickRunner struct { + mu sync.Mutex + calls, active, max int + block chan struct{} +} + +func (r *tickRunner) Tick(ctx context.Context, _ string) (TickResult, error) { + r.mu.Lock() + r.calls++ + r.active++ + if r.active > r.max { + r.max = r.active + } + r.mu.Unlock() + select { + case <-r.block: + case <-ctx.Done(): + } + r.mu.Lock() + r.active-- + r.mu.Unlock() + return TickResult{}, nil +} +func TestWorkerLoopStartStopAndPreventsOverlap(t *testing.T) { + runner := &tickRunner{block: make(chan struct{})} + loop := NewWorkerLoop(runner, LoopConfig{Interval: time.Millisecond, WorkerID: "loop"}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + loop.Start(ctx) + deadline := time.After(time.Second) + for { + runner.mu.Lock() + calls := runner.calls + runner.mu.Unlock() + if calls > 0 { + break + } + select { + case <-deadline: + t.Fatal("loop did not tick") + default: + time.Sleep(time.Millisecond) + } + } + time.Sleep(10 * time.Millisecond) + runner.mu.Lock() + if runner.max != 1 || runner.calls != 1 { + t.Fatalf("calls=%d max=%d", runner.calls, runner.max) + } + runner.mu.Unlock() + close(runner.block) + loop.Stop() + loop.Stop() +} +func TestWorkerLoopTickNowSkipsOverlap(t *testing.T) { + r := &tickRunner{block: make(chan struct{})} + l := NewWorkerLoop(r, LoopConfig{WorkerID: "w"}) + done := make(chan struct{}) + go func() { _, _ = l.Tick(context.Background()); close(done) }() + time.Sleep(time.Millisecond) + if _, ok := l.Tick(context.Background()); ok { + t.Fatal("overlap was not skipped") + } + close(r.block) + <-done +} diff --git a/backend/internal/jobs/provider.go b/backend/internal/jobs/provider.go new file mode 100644 index 0000000..61b8953 --- /dev/null +++ b/backend/internal/jobs/provider.go @@ -0,0 +1,558 @@ +package jobs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "net/url" + "strings" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/prompt" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/providers" +) + +type ProviderRegistry map[string]providers.Adapter + +type ProviderProcessor struct { + Providers ProviderRegistry + Store Store +} + +func (p ProviderProcessor) Advance(ctx context.Context, job Job) (Job, error) { + // A persisted terminal provider result is the recovery checkpoint. Replaying + // it must run downstream idempotent finalizers, never submit/query again. + if job.Status.Terminal() { + return job, nil + } + adapter := p.Providers[job.Provider] + if adapter == nil { + return Job{}, fmt.Errorf("generation provider %q is unavailable", job.Provider) + } + var request providers.Request + if err := json.Unmarshal(job.RequestPayload, &request); err != nil { + return Job{}, errors.New("invalid provider request payload") + } + var result providers.Result + var err error + expectedStatus := job.Status + if job.ProviderTaskID == "" { + if job.ProviderDispatchStartedAt != nil { + failed := StatusFailed + failure := &JobError{Message: "provider submission outcome is unknown; refusing duplicate submission", Retryable: false} + if p.Store == nil { + job.Status, job.Error = failed, failure + return job, nil + } + return p.Store.UpdateJob(ctx, job.ID, workerPatch(job, Patch{Status: &failed, Error: failure})) + } + if p.Store != nil { + started := time.Now().UTC() + prepared, persistErr := p.Store.UpdateJob(ctx, job.ID, workerPatch(job, Patch{ProviderDispatchStartedAt: &started})) + if persistErr != nil { + return Job{}, persistErr + } + job = prepared + } + result, err = adapter.Submit(ctx, request) + } else { + if modeled, ok := adapter.(providers.ModelQueryAdapter); ok { + result, err = modeled.QueryModel(ctx, job.ProviderTaskID, job.ReqKey) + } else { + result, err = adapter.Query(ctx, job.ProviderTaskID) + } + } + if err != nil && job.ProviderTaskID == "" && p.Store != nil { + failed := StatusFailed + failure := &JobError{Message: "provider submission outcome is unknown; refusing duplicate submission", Retryable: false} + return p.Store.UpdateJob(ctx, job.ID, workerPatch(job, Patch{Status: &failed, Error: failure})) + } + if err != nil { + return Job{}, err + } + job.ProviderTaskID = result.TaskID + encoded, err := providers.EncodeResult(result) + if err != nil { + return Job{}, errors.New("encode provider result") + } + job.ResponsePayload = encoded + job.Status = Status(result.Status) + if result.ErrorMessage != "" { + job.Error = &JobError{Message: result.ErrorMessage} + } + if p.Store != nil { + patch := Patch{Status: &job.Status, ProviderTaskID: &job.ProviderTaskID, ResponsePayload: job.ResponsePayload, SetResponsePayload: true, ClearError: result.ErrorMessage == ""} + patch.ExpectedStatuses = []Status{expectedStatus} + if job.LockedBy != "" { + worker := job.LockedBy + patch.ExpectedLockedBy = &worker + } + if job.Error != nil { + patch.Error = job.Error + } + return p.Store.UpdateJob(ctx, job.ID, patch) + } + return job, nil +} + +type ProviderJobBuilder struct { + ImageProvider, VideoProvider string + ImageModel, VideoModel string + ImageEngine, VideoEngine string + ImageEngines map[string]ProviderTarget + VideoEngines map[string]ProviderTarget + NewID func() string +} + +// ProviderTarget is server-owned routing configuration for one UI engine. +// The request may select a known engine, but it cannot supply a provider or +// model directly. +type ProviderTarget struct { + Provider string + Model string + Settings map[string]any +} + +func (b ProviderJobBuilder) Build(_ context.Context, owner, client, capability, idempotency string, body map[string]any) (CreateCommand, error) { + if capability != "image.generate" && capability != "video.generate" { + return CreateCommand{}, &Error{Kind: ErrorInvalid, Status: 400, Message: "Unsupported capability: " + capability} + } + target, engine, err := b.target(capability, body["engine"]) + if err != nil { + return CreateCommand{}, err + } + if target.Provider == "" || target.Model == "" || b.NewID == nil { + return CreateCommand{}, errors.New("provider job builder is not configured") + } + prepared, err := prepareProviderRequest(capability, engine, target.Settings, body) + if err != nil { + return CreateCommand{}, err + } + prepared.request.Model = target.Model + raw, err := json.Marshal(prepared.request) + if err != nil { + return CreateCommand{}, errors.New("encode provider request") + } + priority := NormalizePriority(intFromAny(body["priority"])) + webhook, _ := body["webhookUrl"].(string) + webhook = strings.TrimSpace(webhook) + if client != "" && webhook != "" && !validWebhookURL(webhook) { + return CreateCommand{}, invalidPreparation("webhookUrl must be an HTTP or HTTPS URL") + } + return CreateCommand{Job: Job{ID: b.NewID(), OwnerID: owner, ExternalClientID: client, Capability: capability, Provider: target.Provider, ReqKey: target.Model, Status: StatusQueued, Prompt: prepared.request.Prompt, InputURLs: prepared.request.InputURLs, InputAssetIDs: prepared.assetIDs, OutputAssetIDs: []string{}, RequestPayload: raw, IdempotencyKey: idempotency, Priority: priority, WebhookURL: webhook}, IdempotencyBody: body}, nil +} + +func validWebhookURL(value string) bool { + parsed, err := url.Parse(value) + return err == nil && parsed.IsAbs() && parsed.Host != "" && parsed.User == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") +} + +func (b ProviderJobBuilder) target(capability string, rawEngine any) (ProviderTarget, string, error) { + engine := engineName(rawEngine) + if capability == "image.generate" { + if rawEngine != nil && engine == "" { + return ProviderTarget{}, "", invalidPreparation("unsupported image engine") + } + if engine != "" { + if engine != "jimeng" && engine != "evolink" && engine != "bailian" { + return ProviderTarget{}, "", invalidPreparation("unsupported image engine") + } + target, ok := b.ImageEngines[engine] + if !ok || target.Provider == "" || target.Model == "" { + return ProviderTarget{}, "", errors.New("provider job builder is not configured") + } + return target, engine, nil + } + return ProviderTarget{Provider: b.ImageProvider, Model: b.ImageModel}, firstConfiguredEngine(b.ImageEngine, b.ImageProvider, "image"), nil + } + if rawEngine != nil && engine == "" { + return ProviderTarget{}, "", invalidPreparation("unsupported video engine") + } + if engine != "" { + if engine != "seedance" && engine != "bailian" { + return ProviderTarget{}, "", invalidPreparation("unsupported video engine") + } + target, ok := b.VideoEngines[engine] + if !ok || target.Provider == "" || target.Model == "" { + return ProviderTarget{}, "", errors.New("provider job builder is not configured") + } + return target, engine, nil + } + return ProviderTarget{Provider: b.VideoProvider, Model: b.VideoModel}, firstConfiguredEngine(b.VideoEngine, b.VideoProvider, "video"), nil +} + +func firstConfiguredEngine(configured, provider, capabilityKind string) string { + if engine := engineName(configured); engine != "" { + return engine + } + if capabilityKind == "image" { + switch provider { + case "volcengine-visual": + return "jimeng" + case "evolink", "bailian": + return provider + } + } else if provider == "seedance" || provider == "bailian" { + return provider + } + return "" +} + +type preparedProviderRequest struct { + request providers.Request + assetIDs []string +} + +func prepareProviderRequest(capability, engine string, defaults map[string]any, body map[string]any) (preparedProviderRequest, error) { + materials, assembly, err := preparationMaterials(body, capability) + if err != nil { + return preparedProviderRequest{}, err + } + text := stringValue(body["prompt"]) + if text == "" && assembly != nil { + text = prompt.Assemble(*assembly).Prompt + } + text = strings.TrimSpace(text) + if text == "" { + return preparedProviderRequest{}, invalidPreparation("prompt is required") + } + assetIDs := stringsFromAny(body["inputAssetIds"]) + if len(assetIDs) == 0 { + for _, material := range materials { + if material.ID != "" { + assetIDs = append(assetIDs, material.ID) + } + } + } + var urls []string + var settings map[string]any + if capability == "image.generate" { + urls = stringsFromAny(firstNonNil(body["imageUrls"], body["inputUrls"])) + if len(urls) == 0 { + for _, material := range materials { + if material.Type == "image" { + urls = append(urls, material.URL) + } + } + } + if err := validateImageCoverage(text, materials, len(urls)); err != nil { + return preparedProviderRequest{}, err + } + settings, err = imageSettings(body) + if err == nil && engine == "bailian" { + err = validateBailianImage(urls, settings) + } + } else { + if err := validateMaterialCoverage(text, materials); err != nil { + return preparedProviderRequest{}, err + } + for _, material := range materials { + urls = append(urls, material.URL) + } + settings, err = videoSettings(engine, defaults, body["settings"], materials) + } + if err != nil { + return preparedProviderRequest{}, err + } + providerMaterials := make([]providers.Material, 0, len(materials)) + for _, material := range materials { + materialType := providers.MaterialImage + switch material.Type { + case "video": + materialType = providers.MaterialVideo + case "audio": + materialType = providers.MaterialAudio + } + providerMaterials = append(providerMaterials, providers.Material{ + URL: material.URL, Type: materialType, Role: material.Role, Label: material.Label, + }) + } + return preparedProviderRequest{request: providers.Request{Capability: capability, Prompt: text, InputURLs: urls, Materials: providerMaterials, Settings: settings}, assetIDs: assetIDs}, nil +} + +func validateImageCoverage(text string, materials []prompt.Material, imageURLCount int) error { + required := prompt.ExtractRequirements(text) + if required.Video > 0 || required.Audio > 0 { + return validateMaterialCoverage(text, materials) + } + if required.Image > imageURLCount { + return invalidPreparation(fmt.Sprintf("prompt requires @图片%d but only %d image materials were supplied", required.Image, imageURLCount)) + } + return nil +} + +func preparationMaterials(body map[string]any, capability string) ([]prompt.Material, *prompt.Input, error) { + assemblyRecord, hasAssembly := body["promptAssembly"].(map[string]any) + var assembly *prompt.Input + if hasAssembly { + var decoded prompt.Input + if err := decodeViaJSON(assemblyRecord, &decoded); err != nil { + return nil, nil, invalidPreparation("invalid promptAssembly") + } + if capability == "image.generate" { + decoded.Mode = "image" + } else { + decoded.Mode = "video" + } + assembly = &decoded + } + var materials []prompt.Material + if raw, ok := body["materials"]; ok { + if err := decodeViaJSON(raw, &materials); err != nil { + return nil, nil, invalidPreparation("invalid materials") + } + } else if assembly != nil { + materials = assembly.Materials + } + materials = prompt.NormalizeMaterials(materials) + if assembly != nil { + assembly.Materials = materials + } + return materials, assembly, nil +} + +func validateMaterialCoverage(text string, materials []prompt.Material) error { + required := prompt.ExtractRequirements(text) + available := prompt.Requirements{} + for _, material := range materials { + switch material.Type { + case "video": + available.Video++ + case "audio": + available.Audio++ + default: + available.Image++ + } + } + if required.Image > available.Image { + return invalidPreparation(fmt.Sprintf("prompt requires @图片%d but only %d image materials were supplied", required.Image, available.Image)) + } + if required.Video > available.Video { + return invalidPreparation(fmt.Sprintf("prompt requires @视频%d but only %d video materials were supplied", required.Video, available.Video)) + } + if required.Audio > available.Audio { + return invalidPreparation(fmt.Sprintf("prompt requires @音频%d but only %d audio materials were supplied", required.Audio, available.Audio)) + } + return nil +} + +func imageSettings(body map[string]any) (map[string]any, error) { + out := map[string]any{} + if nested, ok := body["settings"].(map[string]any); ok { + for _, key := range []string{"scale", "width", "height", "min_ratio", "max_ratio", "imageCount", "force_single", "quality"} { + if nested[key] != nil { + out[key] = nested[key] + } + } + } + for _, key := range []string{"scale", "width", "height", "min_ratio", "max_ratio", "imageCount"} { + raw := body[key] + if raw == nil { + raw = out[key] + } + if value, ok := finiteNumber(raw); ok { + out[key] = value + } else if raw != nil && raw != "" { + return nil, invalidPreparation("invalid image parameter: " + key) + } else { + delete(out, key) + } + } + forceSingle := body["force_single"] + if forceSingle == nil { + forceSingle = out["force_single"] + } + if value, ok := forceSingle.(bool); ok { + out["force_single"] = value + } else if forceSingle != nil { + return nil, invalidPreparation("invalid image parameter: force_single") + } else { + delete(out, "force_single") + } + qualityRaw := body["quality"] + if qualityRaw == nil { + qualityRaw = out["quality"] + } + delete(out, "quality") + if raw, ok := qualityRaw.(string); ok { + quality := strings.ToLower(strings.TrimSpace(raw)) + if quality == "low" || quality == "medium" || quality == "high" { + out["quality"] = quality + } + } + if count, ok := out["imageCount"].(float64); ok && (count <= 0 || count > 9 || math.Trunc(count) != count) { + return nil, invalidPreparation("imageCount must be an integer between 1 and 9") + } + return out, nil +} + +func validateBailianImage(urls []string, settings map[string]any) error { + if len(urls) > 9 { + return invalidPreparation("bailian supports at most 9 reference images") + } + width, hasWidth := settings["width"].(float64) + height, hasHeight := settings["height"].(float64) + if !hasWidth && !hasHeight { + return nil + } + if !hasWidth || !hasHeight || math.Trunc(width) != width || math.Trunc(height) != height || width <= 0 || height <= 0 { + return invalidPreparation("bailian image dimensions must be positive integers") + } + pixels := width * height + maximum := float64(4096 * 4096) + if len(urls) > 0 { + maximum = float64(2048 * 2048) + } + if pixels < float64(768*768) || pixels > maximum || width/height < 1.0/8.0 || width/height > 8 { + return invalidPreparation("bailian image dimensions are outside supported size constraints") + } + return nil +} + +func videoSettings(engine string, defaults map[string]any, raw any, materials []prompt.Material) (map[string]any, error) { + input, _ := raw.(map[string]any) + merged := make(map[string]any, len(defaults)+len(input)) + for key, value := range defaults { + merged[key] = value + } + for key, value := range input { + merged[key] = value + } + input = merged + settings := map[string]any{} + if engine == "bailian" { + if len(materials) < 1 || len(materials) > 2 { + return nil, invalidPreparation("bailian video requires 1 or 2 image materials") + } + for _, material := range materials { + if material.Type != "image" { + return nil, invalidPreparation("bailian video requires 1 or 2 image materials") + } + } + duration := float64(10) + if input["duration"] != nil { + var ok bool + duration, ok = finiteNumber(input["duration"]) + if !ok || math.Trunc(duration) != duration || duration < 2 || duration > 15 { + return nil, invalidPreparation("bailian video duration must be an integer between 2 and 15 seconds") + } + } + resolution := strings.ToUpper(stringValue(input["resolution"])) + if resolution == "" { + resolution = "720P" + } + if resolution != "720P" && resolution != "1080P" { + return nil, invalidPreparation("bailian video resolution must be 720P or 1080P") + } + return map[string]any{"duration": duration, "resolution": resolution}, nil + } + if engine != "" && engine != "seedance" { + return nil, invalidPreparation("unsupported video engine") + } + ratio := stringValue(input["ratio"]) + if ratio != "" && !oneOf(ratio, "9:16", "16:9", "1:1", "4:3", "3:4", "21:9", "adaptive") { + return nil, invalidPreparation("unsupported video ratio") + } + if ratio != "" { + settings["ratio"] = ratio + } + if input["duration"] != nil { + duration, ok := finiteNumber(input["duration"]) + if !ok || math.Trunc(duration) != duration || duration < 4 || duration > 15 { + return nil, invalidPreparation("video duration must be an integer between 4 and 15 seconds") + } + settings["duration"] = duration + } + resolution := stringValue(input["resolution"]) + if resolution != "" && !oneOf(resolution, "480p", "720p", "1080p", "4k") { + return nil, invalidPreparation("unsupported video resolution") + } + if resolution != "" { + settings["resolution"] = resolution + } + return settings, nil +} + +func invalidPreparation(message string) error { + return &Error{Kind: ErrorInvalid, Status: 400, Message: message} +} + +func decodeViaJSON(input, output any) error { + raw, err := json.Marshal(input) + if err != nil { + return err + } + return json.Unmarshal(raw, output) +} + +func finiteNumber(value any) (float64, bool) { + var number float64 + switch typed := value.(type) { + case float64: + number = typed + case float32: + number = float64(typed) + case int: + number = float64(typed) + case int64: + number = float64(typed) + default: + return 0, false + } + return number, !math.IsNaN(number) && !math.IsInf(number, 0) +} + +func stringValue(value any) string { + text, _ := value.(string) + return strings.TrimSpace(text) +} + +func engineName(value any) string { return strings.ToLower(stringValue(value)) } + +func oneOf(value string, options ...string) bool { + for _, option := range options { + if value == option { + return true + } + } + return false +} + +func stringsFromAny(v any) []string { + if values, ok := v.([]string); ok { + out := make([]string, 0, len(values)) + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + out = append(out, value) + } + } + return out + } + values, _ := v.([]any) + out := []string{} + for _, x := range values { + if s, ok := x.(string); ok && strings.TrimSpace(s) != "" { + out = append(out, strings.TrimSpace(s)) + } + } + return out +} +func intFromAny(v any) int { + switch x := v.(type) { + case float64: + return int(x) + case int: + return x + } + return 0 +} +func firstNonNil(v ...any) any { + for _, x := range v { + if x != nil { + return x + } + } + return nil +} diff --git a/backend/internal/jobs/provider_test.go b/backend/internal/jobs/provider_test.go new file mode 100644 index 0000000..150676a --- /dev/null +++ b/backend/internal/jobs/provider_test.go @@ -0,0 +1,337 @@ +package jobs + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/providers" +) + +func TestProviderJobBuilderPreparesImageRequestAndEngineOverride(t *testing.T) { + b := testProviderBuilder() + body := map[string]any{ + "engine": "evolink", + "promptAssembly": map[string]any{ + "mode": "image", "manualPrompt": " assembled image ", + "materials": []any{map[string]any{"id": "asset-1", "url": "https://in.test/reference.png", "type": "image"}}, + }, + "scale": 0.7, "width": 1536.0, "height": 1024.0, + "min_ratio": 0.5, "max_ratio": 2.0, "force_single": true, + "quality": " HIGH ", "settings": map[string]any{"imageCount": 2.0}, + } + + cmd, err := b.Build(context.Background(), "owner", "client", "image.generate", "idem", body) + if err != nil { + t.Fatal(err) + } + wantRequest := providers.Request{ + Capability: "image.generate", Model: "gpt-image-test", Prompt: "assembled image", InputURLs: []string{"https://in.test/reference.png"}, + Materials: []providers.Material{{URL: "https://in.test/reference.png", Type: providers.MaterialImage, Label: "@图片1"}}, + Settings: map[string]any{"scale": 0.7, "width": float64(1536), "height": float64(1024), "min_ratio": 0.5, "max_ratio": 2.0, "force_single": true, "quality": "high", "imageCount": float64(2)}, + } + assertPreparedJob(t, cmd.Job, "evolink", "gpt-image-test", wantRequest) + if !reflect.DeepEqual(cmd.Job.InputAssetIDs, []string{"asset-1"}) { + t.Fatalf("InputAssetIDs = %#v", cmd.Job.InputAssetIDs) + } +} + +func TestProviderJobBuilderUsesPublicInputURLsForImageMaterialCoverage(t *testing.T) { + b := testProviderBuilder() + cmd, err := b.Build(context.Background(), "owner", "client", "image.generate", "", map[string]any{ + "prompt": "compose @图片2", "inputUrls": []any{"https://in.test/one.png", "https://in.test/two.png"}, + }) + if err != nil { + t.Fatal(err) + } + var request providers.Request + if err := json.Unmarshal(cmd.Job.RequestPayload, &request); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(request.InputURLs, []string{"https://in.test/one.png", "https://in.test/two.png"}) { + t.Fatalf("InputURLs = %#v", request.InputURLs) + } +} + +func TestProviderJobBuilderPreparesSeedanceVideoRequest(t *testing.T) { + b := testProviderBuilder() + body := map[string]any{ + "engine": "seedance", "prompt": " launch video ", + "materials": []any{ + map[string]any{"id": "image-1", "url": "https://in.test/first.png", "type": "image"}, + map[string]any{"id": "video-1", "url": "https://in.test/reference.mp4", "type": "video"}, + map[string]any{"id": "audio-1", "url": "https://in.test/music.mp3", "type": "audio"}, + }, + "settings": map[string]any{"ratio": "16:9", "duration": 8.0, "resolution": "1080p", "ignored": "value"}, + } + cmd, err := b.Build(context.Background(), "owner", "", "video.generate", "", body) + if err != nil { + t.Fatal(err) + } + wantRequest := providers.Request{ + Capability: "video.generate", Model: "seedance-test", Prompt: "launch video", + InputURLs: []string{"https://in.test/first.png", "https://in.test/reference.mp4", "https://in.test/music.mp3"}, + Materials: []providers.Material{ + {URL: "https://in.test/first.png", Type: providers.MaterialImage, Label: "@图片1"}, + {URL: "https://in.test/reference.mp4", Type: providers.MaterialVideo, Label: "@视频1"}, + {URL: "https://in.test/music.mp3", Type: providers.MaterialAudio, Label: "@音频1"}, + }, + Settings: map[string]any{"ratio": "16:9", "duration": float64(8), "resolution": "1080p"}, + } + assertPreparedJob(t, cmd.Job, "seedance", "seedance-test", wantRequest) +} + +func TestProviderJobBuilderAppliesInjectedVideoDefaults(t *testing.T) { + b := testProviderBuilder() + cmd, err := b.Build(context.Background(), "owner", "", "video.generate", "", map[string]any{ + "engine": "seedance", "prompt": "video", + }) + if err != nil { + t.Fatal(err) + } + var request providers.Request + if err := json.Unmarshal(cmd.Job.RequestPayload, &request); err != nil { + t.Fatal(err) + } + want := map[string]any{"ratio": "9:16", "duration": float64(5), "resolution": "720p"} + if !reflect.DeepEqual(request.Settings, want) { + t.Fatalf("Settings = %#v, want %#v", request.Settings, want) + } +} + +func TestProviderJobBuilderPreparesBailianFirstAndLastFrameVideo(t *testing.T) { + b := testProviderBuilder() + body := map[string]any{ + "engine": "bailian", + "promptAssembly": map[string]any{ + "mode": "video", "manualPrompt": "animate frames", + "materials": []any{ + map[string]any{"id": "first", "url": "https://in.test/first.png", "type": "image"}, + map[string]any{"id": "last", "url": "https://in.test/last.png", "type": "image"}, + }, + }, + "settings": map[string]any{"duration": 12.0, "resolution": "1080p"}, + } + cmd, err := b.Build(context.Background(), "owner", "", "video.generate", "", body) + if err != nil { + t.Fatal(err) + } + wantRequest := providers.Request{ + Capability: "video.generate", Model: "bailian-video-test", Prompt: "animate frames", + InputURLs: []string{"https://in.test/first.png", "https://in.test/last.png"}, + Materials: []providers.Material{ + {URL: "https://in.test/first.png", Type: providers.MaterialImage, Label: "@图片1"}, + {URL: "https://in.test/last.png", Type: providers.MaterialImage, Label: "@图片2"}, + }, + Settings: map[string]any{"duration": float64(12), "resolution": "1080P"}, + } + assertPreparedJob(t, cmd.Job, "bailian", "bailian-video-test", wantRequest) +} + +func TestProviderJobBuilderRejectsInvalidPreparation(t *testing.T) { + tests := []struct { + name, capability, message string + body map[string]any + }{ + {name: "missing image prompt", capability: "image.generate", body: map[string]any{}, message: "prompt is required"}, + {name: "arbitrary provider", capability: "image.generate", body: map[string]any{"prompt": "p", "engine": "attacker-provider"}, message: "unsupported image engine"}, + {name: "bailian too many references", capability: "image.generate", body: map[string]any{"prompt": "p", "engine": "bailian", "imageUrls": tenURLs()}, message: "at most 9 reference images"}, + {name: "bailian invalid image pixels", capability: "image.generate", body: map[string]any{"prompt": "p", "engine": "bailian", "width": 100.0, "height": 100.0}, message: "image dimensions"}, + {name: "seedance missing materials referenced by prompt", capability: "video.generate", body: map[string]any{"engine": "seedance", "prompt": "use @图片2"}, message: "requires @图片2"}, + {name: "bailian requires frame", capability: "video.generate", body: map[string]any{"engine": "bailian", "prompt": "p", "materials": []any{}}, message: "1 or 2 image materials"}, + {name: "bailian rejects video material", capability: "video.generate", body: map[string]any{"engine": "bailian", "prompt": "p", "materials": []any{map[string]any{"url": "https://in.test/a.mp4", "type": "video"}}}, message: "1 or 2 image materials"}, + {name: "bad seedance settings", capability: "video.generate", body: map[string]any{"engine": "seedance", "prompt": "p", "settings": map[string]any{"duration": 99.0}}, message: "video duration"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := testProviderBuilder().Build(context.Background(), "owner", "", test.capability, "", test.body) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.message)) { + t.Fatalf("error = %v, want containing %q", err, test.message) + } + var jobErr *Error + if !strings.Contains(err.Error(), "configured") && (!errorsAs(err, &jobErr) || jobErr.Status != 400 || jobErr.Kind != ErrorInvalid) { + t.Fatalf("error = %#v, want generic invalid job error", err) + } + }) + } +} + +func TestProviderBuilderAndProcessor(t *testing.T) { + b := testProviderBuilder() + cmd, err := b.Build(context.Background(), "owner", "client", "image.generate", "idem", map[string]any{"prompt": "hello", "inputUrls": []any{"https://in.test/a.png"}}) + if err != nil || cmd.Job.OwnerID != "owner" || cmd.Job.ExternalClientID != "client" { + t.Fatalf("Build=%#v,%v", cmd, err) + } + store := newMemoryJobStore() + store.jobs[cmd.Job.ID] = cmd.Job + p := ProviderProcessor{Providers: ProviderRegistry{"image-default": providers.NewMock("fixture")}, Store: store} + submitted, err := p.Advance(context.Background(), cmd.Job) + if err != nil || submitted.ProviderTaskID == "" || submitted.Status != StatusQueued { + t.Fatalf("submit=%#v,%v", submitted, err) + } + done, err := p.Advance(context.Background(), submitted) + if err != nil || done.Status != StatusSucceeded { + t.Fatalf("query=%#v,%v", done, err) + } + if store.jobs[cmd.Job.ID].Status != StatusSucceeded { + t.Fatal("provider result was not persisted") + } + var persisted providers.HTTPResult + if err := json.Unmarshal(done.ResponsePayload, &persisted); err != nil || len(persisted.OutputURLs) != 1 || persisted.OutputURLs[0] == "" { + t.Fatalf("persisted result = %#v, %v raw=%s", persisted, err, done.ResponsePayload) + } +} + +func TestProviderProcessorNeverResubmitsAfterPersistedDispatchIntent(t *testing.T) { + store := newMemoryJobStore() + started := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + job := Job{ID: "job-unknown", OwnerID: "owner", Provider: "mock", Capability: "image.generate", Status: StatusRunning, LockedBy: "worker", ProviderDispatchStartedAt: &started, RequestPayload: json.RawMessage(`{"capability":"image.generate","model":"fixture","prompt":"hello"}`)} + store.jobs[job.ID] = job + adapter := &countingProvider{result: providers.Result{TaskID: "duplicate", Status: providers.StatusQueued}} + processor := ProviderProcessor{Providers: ProviderRegistry{"mock": adapter}, Store: store} + + got, err := processor.Advance(context.Background(), job) + if err != nil || adapter.submits != 0 || got.Status != StatusFailed || got.Error == nil || got.Error.Retryable { + t.Fatalf("got=%#v err=%v submits=%d", got, err, adapter.submits) + } +} + +func TestProviderProcessorClearsTransientErrorAfterSuccessfulPoll(t *testing.T) { + store := newMemoryJobStore() + job := Job{ID: "job-recovered", OwnerID: "owner", Provider: "fixture", ReqKey: "model-a", Capability: "image.generate", Status: StatusQueued, LockedBy: "worker", ProviderTaskID: "provider-task", Error: &JobError{Message: "temporary timeout", Retryable: true}, RequestPayload: json.RawMessage(`{"capability":"image.generate","model":"model-a","prompt":"hello"}`)} + store.jobs[job.ID] = job + adapter := &countingProvider{result: providers.Result{TaskID: "provider-task", Status: providers.StatusSucceeded, OutputURLs: []string{"https://cdn.test/result.png"}}} + processor := ProviderProcessor{Providers: ProviderRegistry{"fixture": adapter}, Store: store} + + got, err := processor.Advance(context.Background(), job) + if err != nil || got.Status != StatusSucceeded || got.Error != nil { + t.Fatalf("got=%#v err=%v", got, err) + } +} + +func TestProviderProcessorQueriesWithPersistedRequestModel(t *testing.T) { + store := newMemoryJobStore() + job := Job{ID: "job-model", OwnerID: "owner", Provider: "fixture", ReqKey: "persisted-model-a", Capability: "image.generate", Status: StatusQueued, LockedBy: "worker", ProviderTaskID: "provider-task", RequestPayload: json.RawMessage(`{"capability":"image.generate","model":"persisted-model-a","prompt":"hello"}`)} + store.jobs[job.ID] = job + adapter := &modelQueryProvider{result: providers.Result{TaskID: "provider-task", Status: providers.StatusRunning}} + processor := ProviderProcessor{Providers: ProviderRegistry{"fixture": adapter}, Store: store} + + if _, err := processor.Advance(context.Background(), job); err != nil { + t.Fatal(err) + } + if adapter.model != "persisted-model-a" { + t.Fatalf("query model = %q", adapter.model) + } +} + +func TestProviderJobBuilderRejectsInvalidPublicWebhookURL(t *testing.T) { + for _, value := range []string{"/internal/callback", "javascript:alert(1)", "https://user:pass@example.test/hook"} { + _, err := testProviderBuilder().Build(context.Background(), "api:client", "client", "image.generate", "", map[string]any{"prompt": "hello", "webhookUrl": value}) + var invalid *Error + if !errorsAs(err, &invalid) || invalid.Status != 400 { + t.Fatalf("webhook %q error = %#v", value, err) + } + } + cmd, err := testProviderBuilder().Build(context.Background(), "api:client", "client", "image.generate", "", map[string]any{"prompt": "hello", "webhookUrl": "https://hooks.example.test/done"}) + if err != nil || cmd.Job.WebhookURL != "https://hooks.example.test/done" { + t.Fatalf("valid webhook = %#v, %v", cmd.Job, err) + } +} + +func TestProviderJobBuilderClampsPriorityToPublicContract(t *testing.T) { + for _, test := range []struct { + raw any + want int + }{{raw: 999.0, want: 100}, {raw: -999.0, want: -100}, {raw: 42.0, want: 42}} { + cmd, err := testProviderBuilder().Build(context.Background(), "owner", "client", "image.generate", "", map[string]any{"prompt": "hello", "priority": test.raw}) + if err != nil || cmd.Job.Priority != test.want { + t.Fatalf("priority(%v)=%d err=%v, want %d", test.raw, cmd.Job.Priority, err, test.want) + } + } +} + +type countingProvider struct { + submits int + result providers.Result +} + +type modelQueryProvider struct { + result providers.Result + model string +} + +func (provider *modelQueryProvider) Submit(context.Context, providers.Request) (providers.Result, error) { + return provider.result, nil +} +func (provider *modelQueryProvider) Query(context.Context, string) (providers.Result, error) { + return providers.Result{}, errors.New("fallback query must not be used") +} +func (provider *modelQueryProvider) QueryModel(_ context.Context, _ string, model string) (providers.Result, error) { + provider.model = model + return provider.result, nil +} + +func (p *countingProvider) Submit(context.Context, providers.Request) (providers.Result, error) { + p.submits++ + return p.result, nil +} +func (p *countingProvider) Query(context.Context, string) (providers.Result, error) { + return p.result, nil +} + +func testProviderBuilder() ProviderJobBuilder { + return ProviderJobBuilder{ + ImageProvider: "image-default", ImageModel: "image-default-model", + VideoProvider: "video-default", VideoModel: "video-default-model", + ImageEngine: "jimeng", VideoEngine: "seedance", + ImageEngines: map[string]ProviderTarget{ + "jimeng": {Provider: "volcengine-visual", Model: "jimeng-test"}, + "evolink": {Provider: "evolink", Model: "gpt-image-test"}, + "bailian": {Provider: "bailian", Model: "bailian-image-test"}, + }, + VideoEngines: map[string]ProviderTarget{ + "seedance": {Provider: "seedance", Model: "seedance-test", Settings: map[string]any{"ratio": "9:16", "duration": 5, "resolution": "720p"}}, + "bailian": {Provider: "bailian", Model: "bailian-video-test"}, + }, + NewID: func() string { return "job-1" }, + } +} + +func assertPreparedJob(t *testing.T, job Job, provider, model string, want providers.Request) { + t.Helper() + if job.Provider != provider || job.ReqKey != model { + t.Fatalf("provider/model = %q/%q, want %q/%q", job.Provider, job.ReqKey, provider, model) + } + var got providers.Request + if err := json.Unmarshal(job.RequestPayload, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("request = %#v, want %#v; raw=%s", got, want, job.RequestPayload) + } + if job.Prompt != want.Prompt || !reflect.DeepEqual(job.InputURLs, want.InputURLs) { + t.Fatalf("job preparation = prompt %q URLs %#v", job.Prompt, job.InputURLs) + } +} + +func tenURLs() []any { + urls := make([]any, 10) + for i := range urls { + urls[i] = "https://in.test/reference.png" + } + return urls +} + +// Kept local so the test remains compatible with the package's supported Go version. +func errorsAs(err error, target any) bool { + jobErr, ok := err.(*Error) + pointer, targetOK := target.(**Error) + if ok && targetOK { + *pointer = jobErr + } + return ok && targetOK +} diff --git a/backend/internal/jobs/service.go b/backend/internal/jobs/service.go index a8fb6ba..d079c19 100644 --- a/backend/internal/jobs/service.go +++ b/backend/internal/jobs/service.go @@ -2,6 +2,7 @@ package jobs import ( "context" + "errors" "fmt" "time" ) @@ -12,21 +13,35 @@ type Store interface { FindIdempotentJob(context.Context, Scope, string) (Job, bool, error) CreateJob(context.Context, Job) (Job, error) UpdateJob(context.Context, string, Patch) (Job, error) + DeleteJob(context.Context, string) error ClaimJobs(context.Context, string, int, int) ([]Job, error) } +type ArtifactDeleter interface { + DeleteOutputs(context.Context, Job) ([]string, error) +} + type Patch struct { - Status *Status - Error *JobError - Attempts *int - ScheduledAt *time.Time - CompletedAt *time.Time - ProviderTaskID *string - ClearProviderTaskID bool - ClearLease bool - WebhookAttempts *int - WebhookLastStatus []byte - SetWebhookStatus bool + Status *Status + Error *JobError + ClearError bool + Attempts *int + ScheduledAt *time.Time + CompletedAt *time.Time + ProviderTaskID *string + ProviderDispatchStartedAt *time.Time + ResponsePayload []byte + SetResponsePayload bool + ClearProviderTaskID bool + ClearProviderDispatch bool + ClearLease bool + WebhookAttempts *int + WebhookLastStatus []byte + SetWebhookStatus bool + FinalizedAt *time.Time + DispatchReadyAt *time.Time + ExpectedStatuses []Status + ExpectedLockedBy *string } type Service struct { @@ -85,6 +100,10 @@ func (service *Service) Create(ctx context.Context, command CreateCommand) (Job, if job.ScheduledAt.IsZero() { job.ScheduledAt = now } + if job.DispatchReadyAt == nil && !command.HoldDispatch { + ready := now + job.DispatchReadyAt = &ready + } if job.InputAssetIDs == nil { job.InputAssetIDs = []string{} } @@ -143,7 +162,18 @@ func (service *Service) Cancel(ctx context.Context, scope Scope, id string, refu } now := service.now().UTC() status := StatusCancelled - job, err = service.store.UpdateJob(ctx, job.ID, Patch{Status: &status, CompletedAt: &now}) + patch := Patch{Status: &status, CompletedAt: &now, ClearLease: true, ExpectedStatuses: []Status{job.Status}} + if job.LockedBy != "" { + owner := job.LockedBy + patch.ExpectedLockedBy = &owner + } + job, err = service.store.UpdateJob(ctx, job.ID, patch) + if errors.Is(err, ErrTransitionConflict) { + current, getErr := service.Get(ctx, scope, id) + if getErr == nil && current.Status.Terminal() { + return current, nil + } + } if err != nil { return Job{}, err } @@ -153,7 +183,41 @@ func (service *Service) Cancel(ctx context.Context, scope Scope, id string, refu return Job{}, err } } - return service.store.UpdateJob(ctx, job.ID, Patch{ClearLease: true}) + return job, nil +} + +func (service *Service) Retry(ctx context.Context, scope Scope, id, newID string) (Job, error) { + original, err := service.Get(ctx, scope, id) + if err != nil { + return Job{}, err + } + if original.Capability != "image.generate" || (original.Status != StatusFailed && original.Status != StatusExpired && original.Status != StatusCancelled) { + return Job{}, &Error{Kind: ErrorInvalid, Status: 400, Message: "任务当前不可重试"} + } + now := service.now().UTC() + retry := original + retry.ID, retry.RetryOf, retry.Status = newID, original.ID, StatusQueued + retry.ProviderTaskID, retry.Error, retry.IdempotencyKey, retry.IdempotencyFingerprint = "", nil, "", "" + retry.OutputAssetIDs, retry.Attempts, retry.LockedAt, retry.LockedBy = []string{}, 0, nil, "" + retry.StartedAt, retry.CompletedAt = nil, nil + retry.ScheduledAt, retry.CreatedAt, retry.UpdatedAt = now, now, now + return service.store.CreateJob(ctx, retry) +} + +func (service *Service) Delete(ctx context.Context, scope Scope, id string, artifacts ArtifactDeleter) (Job, error) { + job, err := service.Get(ctx, scope, id) + if err != nil { + return Job{}, err + } + if artifacts != nil { + if _, err := artifacts.DeleteOutputs(ctx, job); err != nil { + return Job{}, err + } + } + if err := service.store.DeleteJob(ctx, job.ID); err != nil { + return Job{}, err + } + return job, nil } func RetryDelay(attempts int, base, maximum time.Duration) time.Duration { diff --git a/backend/internal/jobs/service_http_test.go b/backend/internal/jobs/service_http_test.go new file mode 100644 index 0000000..25e765b --- /dev/null +++ b/backend/internal/jobs/service_http_test.go @@ -0,0 +1,38 @@ +package jobs + +import ( + "context" + "testing" + "time" +) + +func TestServiceDeletesScopedJobAndRetriesOnlyFailedImage(t *testing.T) { + now := time.Date(2026, 8, 13, 9, 0, 0, 0, time.UTC) + store := newMemoryJobStore() + store.jobs["failed"] = Job{ID: "failed", OwnerID: "user-1", Capability: "image.generate", Provider: "mock", ReqKey: "fixture", Status: StatusFailed, Prompt: "p", InputAssetIDs: []string{}, InputURLs: []string{}, OutputAssetIDs: []string{"asset-1"}, RequestPayload: []byte(`{"prompt":"p"}`), CreatedAt: now, UpdatedAt: now} + service := NewService(store, func() time.Time { return now }) + + retried, err := service.Retry(context.Background(), Scope{OwnerID: "user-1"}, "failed", "retry-1") + if err != nil || retried.ID != "retry-1" || retried.RetryOf != "failed" || retried.Status != StatusQueued || retried.OutputAssetIDs == nil || len(retried.OutputAssetIDs) != 0 { + t.Fatalf("Retry = %#v, %v", retried, err) + } + deleted, err := service.Delete(context.Background(), Scope{OwnerID: "user-1"}, "failed", nil) + if err != nil || deleted.ID != "failed" { + t.Fatalf("Delete = %#v, %v", deleted, err) + } + if _, ok := store.jobs["failed"]; ok { + t.Fatal("job was not deleted") + } +} + +func TestServiceRejectsRetryForVideoOrActiveJob(t *testing.T) { + store := newMemoryJobStore() + store.jobs["video"] = Job{ID: "video", OwnerID: "user-1", Capability: "video.generate", Status: StatusFailed} + store.jobs["active"] = Job{ID: "active", OwnerID: "user-1", Capability: "image.generate", Status: StatusRunning} + service := NewService(store, nil) + for _, id := range []string{"video", "active"} { + if _, err := service.Retry(context.Background(), Scope{OwnerID: "user-1"}, id, "new"); err == nil { + t.Fatalf("Retry(%s) error=nil", id) + } + } +} diff --git a/backend/internal/jobs/worker.go b/backend/internal/jobs/worker.go index 131e106..69f0571 100644 --- a/backend/internal/jobs/worker.go +++ b/backend/internal/jobs/worker.go @@ -2,6 +2,7 @@ package jobs import ( "context" + "crypto/rand" "encoding/json" "fmt" "time" @@ -37,13 +38,14 @@ type WorkerConfig struct { } type Worker struct { - store Store - processor Processor - refunds TerminalRefund - usage UsageRecorder - webhooks WebhookDelivery - config WorkerConfig - now func() time.Time + store Store + processor Processor + refunds TerminalRefund + usage UsageRecorder + webhooks WebhookDelivery + config WorkerConfig + now func() time.Time + leaseOwner func(string) (string, error) } func NewWorker(store Store, processor Processor, refunds TerminalRefund, usage UsageRecorder, webhooks WebhookDelivery, config WorkerConfig, now func() time.Time) *Worker { @@ -62,7 +64,7 @@ func NewWorker(store Store, processor Processor, refunds TerminalRefund, usage U if config.PollInterval <= 0 { config.PollInterval = 5 * time.Second } - return &Worker{store: store, processor: processor, refunds: refunds, usage: usage, webhooks: webhooks, config: config, now: now} + return &Worker{store: store, processor: processor, refunds: refunds, usage: usage, webhooks: webhooks, config: config, now: now, leaseOwner: newLeaseOwner} } type TickResult struct { @@ -79,17 +81,35 @@ type TickJob struct { } func (worker *Worker) Tick(ctx context.Context, workerID string) (TickResult, error) { - claimed, err := worker.store.ClaimJobs(ctx, workerID, worker.config.BatchSize, worker.config.LockTimeoutSeconds) + return worker.TickLimit(ctx, workerID, worker.config.BatchSize) +} + +// TickLimit preserves the legacy internal HTTP tick's per-request bounded limit. +func (worker *Worker) TickLimit(ctx context.Context, workerID string, limit int) (TickResult, error) { + if limit <= 0 { + limit = worker.config.BatchSize + } + limit = max(1, min(limit, 20)) + leaseOwner, err := worker.leaseOwner(workerID) + if err != nil { + return TickResult{}, fmt.Errorf("create worker lease owner: %w", err) + } + claimed, err := worker.store.ClaimJobs(ctx, leaseOwner, limit, worker.config.LockTimeoutSeconds) if err != nil { return TickResult{}, err } result := TickResult{WorkerID: workerID, Claimed: len(claimed), Jobs: make([]TickJob, 0, len(claimed))} for _, job := range claimed { advanced, advanceErr := worker.processor.Advance(ctx, job) + // Adapters must preserve the lease token on returned snapshots so every + // downstream settlement/output/refund write remains fenced. + if advanced.ID != "" && advanced.LockedBy == "" { + advanced.LockedBy, advanced.LockedAt = job.LockedBy, job.LockedAt + } if advanceErr != nil { status := StatusFailed jobError := &JobError{Message: advanceErr.Error(), Retryable: true} - advanced, err = worker.store.UpdateJob(ctx, job.ID, Patch{Status: &status, Error: jobError}) + advanced, err = worker.store.UpdateJob(ctx, job.ID, workerPatch(job, Patch{Status: &status, Error: jobError})) if err != nil { return result, err } @@ -110,18 +130,29 @@ func (worker *Worker) Tick(ctx context.Context, workerID string) (TickResult, er func (worker *Worker) settle(ctx context.Context, job Job) (Job, string, error) { now := worker.now().UTC() + leasePatch := func(patch Patch) Patch { + patch.ExpectedStatuses = []Status{job.Status} + if job.LockedBy != "" { + leaseOwner := job.LockedBy + patch.ExpectedLockedBy = &leaseOwner + } + return patch + } if job.Status == StatusFailed && job.Error != nil && job.Error.Retryable && job.Attempts < maxAttempts(job) { attempts := job.Attempts + 1 scheduled := now.Add(RetryDelay(attempts, worker.config.RetryBase, worker.config.RetryMaximum)) status := StatusQueued - returnPatch := Patch{Status: &status, Attempts: &attempts, ScheduledAt: &scheduled, ClearProviderTaskID: true, ClearLease: true} + // A known provider task is the recovery handle. Retrying a poll must keep + // it; clearing it would turn a transient query failure into a duplicate + // provider submission. + returnPatch := leasePatch(Patch{Status: &status, Attempts: &attempts, ScheduledAt: &scheduled, ClearError: true, ClearLease: true}) retried, err := worker.store.UpdateJob(ctx, job.ID, returnPatch) return retried, "retry_scheduled", err } if !job.Status.Terminal() { scheduled := now.Add(worker.config.PollInterval) - released, err := worker.store.UpdateJob(ctx, job.ID, Patch{ScheduledAt: &scheduled, ClearLease: true}) + released, err := worker.store.UpdateJob(ctx, job.ID, leasePatch(Patch{ScheduledAt: &scheduled, ClearLease: true})) return released, "released", err } @@ -137,14 +168,11 @@ func (worker *Worker) settle(ctx context.Context, job Job) (Job, string, error) return Job{}, "", err } } - attempts := job.Attempts - if job.Status == StatusFailed { - attempts++ - } completed := now - job, err := worker.store.UpdateJob(ctx, job.ID, Patch{Attempts: &attempts, CompletedAt: &completed, ClearLease: true}) - if err != nil { - return Job{}, "", err + finalPatch := Patch{CompletedAt: &completed, FinalizedAt: &completed, ClearLease: true} + if job.Status == StatusFailed { + attempts := job.Attempts + 1 + finalPatch.Attempts = &attempts } if worker.webhooks != nil { delivery, err := worker.webhooks.Deliver(ctx, job) @@ -156,15 +184,36 @@ func (worker *Worker) settle(ctx context.Context, job Job) (Job, string, error) if err != nil { return Job{}, "", fmt.Errorf("encode webhook last status: %w", err) } - job, err = worker.store.UpdateJob(ctx, job.ID, Patch{WebhookAttempts: &delivery.Attempts, WebhookLastStatus: lastStatus, SetWebhookStatus: true}) + job, err = worker.store.UpdateJob(ctx, job.ID, leasePatch(Patch{WebhookAttempts: &delivery.Attempts, WebhookLastStatus: lastStatus, SetWebhookStatus: true})) if err != nil { return Job{}, "", err } } } + job, err := worker.store.UpdateJob(ctx, job.ID, leasePatch(finalPatch)) + if err != nil { + return Job{}, "", err + } return job, "processed", nil } +func newLeaseOwner(workerID string) (string, error) { + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return "", err + } + return fmt.Sprintf("%s:%x", workerID, nonce), nil +} + +func workerPatch(job Job, patch Patch) Patch { + patch.ExpectedStatuses = []Status{job.Status} + if job.LockedBy != "" { + worker := job.LockedBy + patch.ExpectedLockedBy = &worker + } + return patch +} + func maxAttempts(job Job) int { if job.MaxAttempts > 0 { return job.MaxAttempts diff --git a/backend/internal/jobs/worker_test.go b/backend/internal/jobs/worker_test.go new file mode 100644 index 0000000..082ec84 --- /dev/null +++ b/backend/internal/jobs/worker_test.go @@ -0,0 +1,104 @@ +package jobs + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestWorkerUsesUniqueLeaseOwnerForEachTick(t *testing.T) { + store := &leaseTestStore{} + worker := NewWorker(store, nil, nil, nil, nil, WorkerConfig{}, nil) + tokens := []string{"worker-1:lease-a", "worker-1:lease-b"} + worker.leaseOwner = func(workerID string) (string, error) { + token := tokens[0] + tokens = tokens[1:] + return token, nil + } + + first, err := worker.Tick(context.Background(), "worker-1") + if err != nil { + t.Fatal(err) + } + second, err := worker.Tick(context.Background(), "worker-1") + if err != nil { + t.Fatal(err) + } + + if first.WorkerID != "worker-1" || second.WorkerID != "worker-1" { + t.Fatalf("public worker IDs = %q, %q", first.WorkerID, second.WorkerID) + } + if len(store.claimOwners) != 2 || store.claimOwners[0] == store.claimOwners[1] { + t.Fatalf("claim owners = %#v, want two unique lease tokens", store.claimOwners) + } + if store.claimOwners[0] != "worker-1:lease-a" || store.claimOwners[1] != "worker-1:lease-b" { + t.Fatalf("claim owners = %#v", store.claimOwners) + } +} + +func TestWorkerSettleUsesClaimedJobLeaseToken(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + store := &leaseTestStore{job: Job{ID: "job-1", Status: StatusRunning, LockedBy: "worker-1:lease-current"}} + worker := NewWorker(store, nil, nil, nil, nil, WorkerConfig{}, func() time.Time { return now }) + + settled, action, err := worker.settle(context.Background(), store.job) + if err != nil || action != "released" { + t.Fatalf("settle = (%#v, %q, %v)", settled, action, err) + } + if len(store.expectedLeaseOwners) != 1 || store.expectedLeaseOwners[0] != "worker-1:lease-current" { + t.Fatalf("expected lease owners = %#v", store.expectedLeaseOwners) + } +} + +func TestWorkerStaleLeaseCannotSettleNewLeaseFromSameWorkerID(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + store := &leaseTestStore{job: Job{ID: "job-1", Status: StatusRunning, LockedBy: "worker-1:lease-new"}} + worker := NewWorker(store, nil, nil, nil, nil, WorkerConfig{}, func() time.Time { return now }) + stale := store.job + stale.LockedBy = "worker-1:lease-old" + + _, _, err := worker.settle(context.Background(), stale) + if !errors.Is(err, ErrTransitionConflict) { + t.Fatalf("settle stale lease error = %v, want transition conflict", err) + } + if store.job.LockedBy != "worker-1:lease-new" { + t.Fatalf("fresh lease was overwritten: %#v", store.job) + } +} + +type leaseTestStore struct { + job Job + claimOwners []string + expectedLeaseOwners []string +} + +func (store *leaseTestStore) ListJobs(context.Context, ListFilter) ([]Job, error) { return nil, nil } +func (store *leaseTestStore) FindJob(context.Context, Scope, string) (Job, bool, error) { + return Job{}, false, nil +} +func (store *leaseTestStore) FindIdempotentJob(context.Context, Scope, string) (Job, bool, error) { + return Job{}, false, nil +} +func (store *leaseTestStore) CreateJob(context.Context, Job) (Job, error) { return Job{}, nil } +func (store *leaseTestStore) DeleteJob(context.Context, string) error { return nil } +func (store *leaseTestStore) ClaimJobs(_ context.Context, owner string, _ int, _ int) ([]Job, error) { + store.claimOwners = append(store.claimOwners, owner) + return nil, nil +} +func (store *leaseTestStore) UpdateJob(_ context.Context, _ string, patch Patch) (Job, error) { + if patch.ExpectedLockedBy != nil { + store.expectedLeaseOwners = append(store.expectedLeaseOwners, *patch.ExpectedLockedBy) + if store.job.LockedBy != *patch.ExpectedLockedBy { + return Job{}, ErrTransitionConflict + } + } + if patch.ScheduledAt != nil { + store.job.ScheduledAt = *patch.ScheduledAt + } + if patch.ClearLease { + store.job.LockedBy = "" + store.job.LockedAt = nil + } + return store.job, nil +} diff --git a/backend/internal/localstore/accounts.go b/backend/internal/localstore/accounts.go new file mode 100644 index 0000000..9cbf4cc --- /dev/null +++ b/backend/internal/localstore/accounts.go @@ -0,0 +1,292 @@ +package localstore + +import ( + "context" + "crypto/subtle" + "encoding/hex" + "fmt" + "sort" + "strings" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "golang.org/x/crypto/scrypt" +) + +func (s *Store) ListAccounts(_ context.Context, f administration.AccountFilters) ([]administration.Account, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []administration.Account{} + for _, a := range s.accounts { + if f.OrganizationID != "" && a.OrganizationID != f.OrganizationID || f.Role != "" && a.Role != f.Role || !f.IncludeDisabled && a.Status == administration.StatusDisabled { + continue + } + out = append(out, a) + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.Before(out[j].CreatedAt) }) + return out, nil +} +func (s *Store) GetAccount(_ context.Context, id string) (administration.Account, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + a, ok := s.accounts[id] + return a, ok, nil +} +func (s *Store) CreateAccount(_ context.Context, a administration.Account) (administration.Account, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.accounts[a.ID]; ok { + return administration.Account{}, conflict("账号已存在。") + } + for _, v := range s.accounts { + if administration.NormalizePhone(v.Phone) == administration.NormalizePhone(a.Phone) { + return administration.Account{}, conflict("手机号已存在。") + } + } + s.accounts[a.ID] = a + return a, nil +} +func (s *Store) UpdateAccount(_ context.Context, a administration.Account) (administration.Account, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.accounts[a.ID]; !ok { + return administration.Account{}, notFound("账号不存在。") + } + for id, v := range s.accounts { + if id != a.ID && administration.NormalizePhone(v.Phone) == administration.NormalizePhone(a.Phone) { + return administration.Account{}, conflict("手机号已存在。") + } + } + s.accounts[a.ID] = a + return a, nil +} +func (s *Store) DeleteAccount(_ context.Context, id, archive string) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.accounts[id]; !ok { + return notFound("账号不存在。") + } + if archive != "" { + for key, a := range s.assets { + if a.OwnerID == id { + a.OwnerID = archive + s.assets[key] = a + } + } + for key, j := range s.jobs { + if j.OwnerID == id { + j.OwnerID = archive + s.jobs[key] = j + } + } + for key, t := range s.templates { + if t.OwnerID == id { + t.OwnerID = archive + s.templates[key] = t + } + } + for key, event := range s.usageEvents { + if event.OwnerID == id { + event.OwnerID = archive + s.usageEvents[key] = event + } + } + } + delete(s.accounts, id) + return nil +} +func (s *Store) ListOrganizations(_ context.Context, include bool) ([]administration.Organization, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []administration.Organization{} + for _, o := range s.organizations { + if include || o.Status != administration.StatusDisabled { + out = append(out, o) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.Before(out[j].CreatedAt) }) + return out, nil +} +func (s *Store) GetOrganization(_ context.Context, id string) (administration.Organization, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + o, ok := s.organizations[id] + return o, ok, nil +} +func (s *Store) CreateOrganization(_ context.Context, o administration.Organization) (administration.Organization, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.organizations[o.ID]; ok { + return administration.Organization{}, conflict("组织已存在。") + } + s.organizations[o.ID] = o + return o, nil +} +func (s *Store) UpdateOrganization(_ context.Context, o administration.Organization) (administration.Organization, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.organizations[o.ID]; !ok { + return administration.Organization{}, notFound("组织不存在。") + } + s.organizations[o.ID] = o + return o, nil +} +func (s *Store) DeleteOrganization(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.organizations[id]; !ok { + return notFound("组织不存在。") + } + for _, a := range s.accounts { + if a.OrganizationID == id { + return conflict("组织仍有成员。") + } + } + delete(s.organizations, id) + delete(s.wallets, id) + return nil +} +func (s *Store) CountOrganizationMembers(_ context.Context, id string) (int, error) { + s.mu.RLock() + defer s.mu.RUnlock() + n := 0 + for _, a := range s.accounts { + if a.OrganizationID == id { + n++ + } + } + return n, nil +} +func conflict(message string) error { + return &administration.Error{Kind: administration.ErrorConflict, Message: message} +} +func notFound(message string) error { + return &administration.Error{Kind: administration.ErrorNotFound, Message: message} +} + +func (s *Store) FindAuthorizationSnapshot(_ context.Context, key string) (identity.AuthorizationSnapshot, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + for _, a := range s.accounts { + if a.ID == key || a.LegacySubject == key { + return s.snapshotLocked(a), true, nil + } + } + return identity.AuthorizationSnapshot{}, false, nil +} +func (s *Store) snapshotLocked(a administration.Account) identity.AuthorizationSnapshot { + out := identity.AuthorizationSnapshot{Account: identity.AccountSnapshot{ID: a.ID, Phone: a.Phone, DisplayName: a.DisplayName, Role: string(a.Role), OrganizationID: a.OrganizationID, Status: string(a.Status), SessionVersion: a.SessionVersion}} + if o, ok := s.organizations[a.OrganizationID]; ok { + out.Organization = &identity.OrganizationSnapshot{ID: o.ID, Name: o.Name, Status: string(o.Status)} + } + return out +} + +const maxFailures = 5 +const lockTime = 15 * time.Minute + +func (s *Store) AttemptPasswordLogin(_ context.Context, phone, password string, now time.Time) (identity.LoginAccount, error) { + s.mu.Lock() + defer s.mu.Unlock() + var a administration.Account + found := false + phone = administration.NormalizePhone(phone) + for _, v := range s.accounts { + if administration.NormalizePhone(v.Phone) == phone { + a = v + found = true + break + } + } + if !found { + return identity.LoginAccount{}, identity.NewPasswordLoginError(identity.LoginFailureInvalidCredentials) + } + if a.Status != administration.StatusActive { + return identity.LoginAccount{}, identity.NewPasswordLoginError(identity.LoginFailureAccountDisabled) + } + if a.LockedUntil != nil && a.LockedUntil.After(now) { + return identity.LoginAccount{}, identity.NewPasswordLoginError(identity.LoginFailureAccountLocked) + } + if !verifyPassword(password, a.PasswordHash, a.PasswordSalt) { + a.FailedLoginCount++ + reason := identity.LoginFailureInvalidCredentials + if a.FailedLoginCount >= maxFailures { + a.FailedLoginCount = 0 + locked := now.Add(lockTime) + a.LockedUntil = &locked + reason = identity.LoginFailureAccountLocked + } + a.UpdatedAt = now + s.accounts[a.ID] = a + return identity.LoginAccount{}, identity.NewPasswordLoginError(reason) + } + a.FailedLoginCount = 0 + a.LockedUntil = nil + loginAt := now + a.LastLoginAt = &loginAt + a.UpdatedAt = now + s.accounts[a.ID] = a + snapshot := s.snapshotLocked(a) + return identity.LoginAccount{Account: snapshot.Account, Organization: snapshot.Organization}, nil +} +func (s *Store) ChangeOwnPassword(_ context.Context, id, current, next string, now time.Time) (identity.AuthorizationSnapshot, error) { + s.mu.Lock() + defer s.mu.Unlock() + a, ok := s.accounts[id] + if !ok || a.Status != administration.StatusActive { + return identity.AuthorizationSnapshot{}, &identity.PasswordChangeError{Reason: identity.PasswordChangeNotFound} + } + if !verifyPassword(current, a.PasswordHash, a.PasswordSalt) { + return identity.AuthorizationSnapshot{}, &identity.PasswordChangeError{Reason: identity.PasswordChangeCurrentIncorrect} + } + hashed, err := administration.HashPassword(next) + if err != nil { + return identity.AuthorizationSnapshot{}, fmt.Errorf("hash new password: %w", err) + } + a.PasswordHash = hashed.Hash + a.PasswordSalt = hashed.Salt + a.SessionVersion++ + a.UpdatedAt = now + s.accounts[id] = a + return s.snapshotLocked(a), nil +} +func verifyPassword(password, hash, salt string) bool { + if saltBytes, err := hex.DecodeString(salt); err != nil || len(saltBytes) == 0 { + return false + } + expected, err := hex.DecodeString(hash) + if err != nil || len(expected) == 0 { + return false + } + derived, err := scrypt.Key([]byte(strings.TrimSpace(password)), []byte(salt), 16384, 8, 1, len(expected)) + return err == nil && subtle.ConstantTimeCompare(derived, expected) == 1 +} + +func (s *Store) BillingOrganizations(_ context.Context) ([]billing.Organization, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []billing.Organization{} + for _, o := range s.organizations { + out = append(out, billing.Organization{ID: o.ID, Name: o.Name, Status: string(o.Status), ArchiveOwnerID: o.ArchiveOwnerID, CreatedAt: o.CreatedAt.Format(time.RFC3339Nano), UpdatedAt: o.UpdatedAt.Format(time.RFC3339Nano)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt }) + return out, nil +} +func (s *Store) BillingMembers(_ context.Context) ([]billing.Member, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []billing.Member{} + for _, a := range s.accounts { + out = append(out, billing.Member{ID: a.ID, DisplayName: a.DisplayName, Phone: a.Phone, Role: string(a.Role), OrganizationID: a.OrganizationID, Status: string(a.Status)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} +func (s *Store) BillingOrganizationExists(_ context.Context, id string) (bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.organizations[id] + return ok, nil +} diff --git a/backend/internal/localstore/billing.go b/backend/internal/localstore/billing.go new file mode 100644 index 0000000..613a019 --- /dev/null +++ b/backend/internal/localstore/billing.go @@ -0,0 +1,167 @@ +package localstore + +import ( + "context" + "fmt" + "sort" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" +) + +func (s *Store) BillingWallet(_ context.Context, organizationID string) (billing.Wallet, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if w, ok := s.wallets[organizationID]; ok { + return w, nil + } + return billing.Wallet{OrganizationID: organizationID, Currency: billing.CurrencyCNY}, nil +} +func (s *Store) BillingWallets(_ context.Context) ([]billing.Wallet, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]billing.Wallet, 0, len(s.wallets)) + for _, w := range s.wallets { + out = append(out, w) + } + sort.Slice(out, func(i, j int) bool { return out[i].UpdatedAt > out[j].UpdatedAt }) + return out, nil +} +func (s *Store) BillingLedger(_ context.Context, org, account string, limit int) ([]billing.LedgerEntry, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if limit <= 0 || limit > 500 { + limit = 500 + } + out := []billing.LedgerEntry{} + for i := len(s.ledger) - 1; i >= 0 && len(out) < limit; i-- { + e := s.ledger[i] + if org != "" && e.OrganizationID != org || account != "" && e.AccountID != account { + continue + } + e.Metadata = cloneMap(e.Metadata) + out = append(out, e) + } + return out, nil +} +func (s *Store) ListBillingPriceRules(_ context.Context, include bool) ([]billing.PriceRule, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []billing.PriceRule{} + for _, r := range s.priceRules { + if include || r.Enabled { + out = append(out, cloneRule(r)) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Provider != out[j].Provider { + return out[i].Provider < out[j].Provider + } + if out[i].Capability != out[j].Capability { + return out[i].Capability < out[j].Capability + } + return out[i].ID < out[j].ID + }) + return out, nil +} +func (s *Store) GetBillingPriceRule(_ context.Context, id string) (*billing.PriceRule, error) { + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.priceRules[id] + if !ok { + return nil, nil + } + r = cloneRule(r) + return &r, nil +} +func (s *Store) UpdateBillingPriceRule(_ context.Context, id string, p billing.PricePatch) (*billing.PriceRule, error) { + s.mu.Lock() + defer s.mu.Unlock() + r, ok := s.priceRules[id] + if !ok { + return nil, nil + } + if p.DimensionKey == "" { + r.MarkupMultiplier = p.MarkupMultiplier + } else { + for di := range r.Dimensions { + if r.Dimensions[di].Key != p.DimensionKey { + continue + } + for ti := range r.Dimensions[di].Tiers { + if fmt.Sprint(r.Dimensions[di].Tiers[ti].Value) == p.TierValue { + r.Dimensions[di].Tiers[ti].MarkupMultiplier = p.MarkupMultiplier + } + } + } + } + r.UpdatedAt = s.now().UTC().Format(time.RFC3339Nano) + s.priceRules[id] = cloneRule(r) + r = cloneRule(r) + return &r, nil +} +func (s *Store) SeedBillingPriceRules(_ context.Context, rules []billing.PriceRule) error { + s.mu.Lock() + defer s.mu.Unlock() + now := s.now().UTC().Format(time.RFC3339Nano) + for _, r := range rules { + existing, ok := s.priceRules[r.ID] + if ok { + r.MarkupMultiplier = existing.MarkupMultiplier + r.CreatedAt = existing.CreatedAt + } + if r.CreatedAt == "" { + r.CreatedAt = now + } + r.UpdatedAt = now + s.priceRules[r.ID] = cloneRule(r) + } + return nil +} + +func (s *Store) PostBillingWalletEntry(ctx context.Context, p billing.WalletPostParams) (billing.WalletPosting, error) { + return s.PostWalletEntry(ctx, p) +} +func (s *Store) PostWalletEntry(_ context.Context, p billing.WalletPostParams) (billing.WalletPosting, error) { + s.mu.Lock() + defer s.mu.Unlock() + if old, ok := s.postings[p.IdempotencyKey]; ok { + if !walletParamsEqual(old.params, p) { + return billing.WalletPosting{}, fmt.Errorf("BILLING_IDEMPOTENCY_PAYLOAD_MISMATCH") + } + return old.posting, nil + } + w := s.wallets[p.OrganizationID] + w.OrganizationID = p.OrganizationID + if w.Currency == "" { + w.Currency = billing.CurrencyCNY + } + next := w.BalanceFen + p.DeltaFen + if next < 0 { + return billing.WalletPosting{}, fmt.Errorf("BILLING_INSUFFICIENT_BALANCE") + } + now := s.now().UTC() + w.BalanceFen = next + if p.DeltaFen > 0 && (p.Kind == "recharge" || p.Kind == "adjustment") { + w.TotalRechargedFen += p.DeltaFen + } + if p.Kind == "charge" && p.DeltaFen < 0 { + w.TotalChargedFen += -p.DeltaFen + } + w.UpdatedAt = now.Format(time.RFC3339Nano) + s.wallets[p.OrganizationID] = w + posting := billing.WalletPosting{LedgerID: p.LedgerID, BalanceAfterFen: next, BalanceFen: next, TotalRechargedFen: w.TotalRechargedFen, TotalChargedFen: w.TotalChargedFen, DeltaFen: p.DeltaFen, CreatedAt: now, UpdatedAt: now} + s.ledger = append(s.ledger, billing.LedgerEntry{ID: p.LedgerID, OrganizationID: p.OrganizationID, AccountID: p.AccountID, JobID: p.JobID, Kind: p.Kind, DeltaFen: p.DeltaFen, BalanceAfterFen: next, Currency: defaultCurrency(p.Currency), IdempotencyKey: p.IdempotencyKey, Description: p.Description, Metadata: cloneMap(p.Metadata), CreatedAt: now.Format(time.RFC3339Nano)}) + p.Metadata = cloneMap(p.Metadata) + s.postings[p.IdempotencyKey] = walletRecord{params: p, posting: posting} + return posting, nil +} +func walletParamsEqual(a, b billing.WalletPostParams) bool { + return a.OrganizationID == b.OrganizationID && a.AccountID == b.AccountID && a.JobID == b.JobID && a.Kind == b.Kind && a.DeltaFen == b.DeltaFen && defaultCurrency(a.Currency) == defaultCurrency(b.Currency) +} +func defaultCurrency(v string) string { + if v == "" { + return billing.CurrencyCNY + } + return v +} diff --git a/backend/internal/localstore/content.go b/backend/internal/localstore/content.go new file mode 100644 index 0000000..41d52f0 --- /dev/null +++ b/backend/internal/localstore/content.go @@ -0,0 +1,110 @@ +package localstore + +import ( + "context" + "sort" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage" +) + +func (s *Store) ListTemplates(_ context.Context, owner string) ([]templates.Template, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []templates.Template{} + for _, v := range s.templates { + if v.OwnerID == owner { + out = append(out, v) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].SortOrder != out[j].SortOrder { + return out[i].SortOrder < out[j].SortOrder + } + return out[i].UpdatedAt.After(out[j].UpdatedAt) + }) + return out, nil +} +func (s *Store) CreateTemplate(_ context.Context, v templates.Template) (templates.Template, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.templates[v.ID]; ok { + return templates.Template{}, templates.ErrInvalidTemplate + } + s.templates[v.ID] = v + return v, nil +} +func (s *Store) UpdateTemplate(_ context.Context, owner, id string, p templates.Patch, now time.Time) (templates.Template, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.templates[id] + if !ok || v.OwnerID != owner { + return templates.Template{}, false, nil + } + if p.Name != nil { + v.Name = *p.Name + } + if p.Description != nil { + v.Description = *p.Description + } + if p.Prompt != nil { + v.Prompt = *p.Prompt + } + if p.PreviewImageURL != nil { + v.PreviewImageURL = *p.PreviewImageURL + } + if p.Settings != nil { + v.Settings = *p.Settings + } + if p.SortOrder != nil { + v.SortOrder = *p.SortOrder + } + v.UpdatedAt = now + s.templates[id] = v + return v, true, nil +} +func (s *Store) DeleteTemplate(_ context.Context, owner, id string) (templates.Template, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.templates[id] + if !ok || v.OwnerID != owner { + return templates.Template{}, false, nil + } + delete(s.templates, id) + return v, true, nil +} + +func (s *Store) Insert(e usage.Event) (usage.Event, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if id, ok := s.usageJobIDs[e.JobID]; ok { + return s.usageEvents[id], false, nil + } + if _, ok := s.usageEvents[e.ID]; ok { + return usage.Event{}, false, nil + } + s.usageEvents[e.ID] = e + if e.JobID != "" { + s.usageJobIDs[e.JobID] = e.ID + } + return e, true, nil +} +func (s *Store) List(f usage.Filters) ([]usage.Event, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []usage.Event{} + for _, e := range s.usageEvents { + if f.OwnerID != "" && e.OwnerID != f.OwnerID || f.OrganizationID != "" && e.OrganizationID != f.OrganizationID || f.Capability != "" && e.Capability != f.Capability || f.Provider != "" && e.Provider != f.Provider || f.From != "" && e.CreatedAt < f.From || f.To != "" && e.CreatedAt >= f.To { + continue + } + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { + if out[i].CreatedAt != out[j].CreatedAt { + return out[i].CreatedAt > out[j].CreatedAt + } + return out[i].ID > out[j].ID + }) + return out, nil +} diff --git a/backend/internal/localstore/store.go b/backend/internal/localstore/store.go new file mode 100644 index 0000000..9cdc563 --- /dev/null +++ b/backend/internal/localstore/store.go @@ -0,0 +1,550 @@ +// Package localstore provides process-local development persistence. +// It is intentionally non-durable and must not be used as a production store. +package localstore + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + "sync" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage" +) + +const ( + DemoAccountID = "demo-merchant" + DemoOrganizationID = "org-demo" + DemoPhone = "13800000000" + DemoPassword = "demo-password" +) + +type Option func(*Store) + +func WithClock(now func() time.Time) Option { + return func(s *Store) { + if now != nil { + s.now = now + } + } +} + +type Store struct { + mu sync.RWMutex + now func() time.Time + accounts map[string]administration.Account + organizations map[string]administration.Organization + assets map[string]assets.Asset + jobs map[string]jobs.Job + templates map[string]templates.Template + usageEvents map[string]usage.Event + usageJobIDs map[string]string + priceRules map[string]billing.PriceRule + wallets map[string]billing.Wallet + ledger []billing.LedgerEntry + postings map[string]walletRecord +} + +type walletRecord struct { + params billing.WalletPostParams + posting billing.WalletPosting +} + +func New(options ...Option) *Store { + s := &Store{now: time.Now, accounts: map[string]administration.Account{}, organizations: map[string]administration.Organization{}, assets: map[string]assets.Asset{}, jobs: map[string]jobs.Job{}, templates: map[string]templates.Template{}, usageEvents: map[string]usage.Event{}, usageJobIDs: map[string]string{}, priceRules: map[string]billing.PriceRule{}, wallets: map[string]billing.Wallet{}, postings: map[string]walletRecord{}} + for _, option := range options { + option(s) + } + s.seedDemo() + return s +} + +func NewStore(options ...Option) *Store { return New(options...) } + +func (s *Store) seedDemo() { + now := s.now().UTC() + password, err := administration.HashPassword(DemoPassword) + if err != nil { + panic(fmt.Sprintf("localstore: hash demo password: %v", err)) + } + s.organizations[DemoOrganizationID] = administration.Organization{ID: DemoOrganizationID, Name: "演示组织", Status: administration.StatusActive, ArchiveOwnerID: DemoAccountID, CreatedAt: now, UpdatedAt: now} + s.accounts[DemoAccountID] = administration.Account{ID: DemoAccountID, Phone: DemoPhone, DisplayName: "智念用户", Role: administration.RoleSuperAdmin, OrganizationID: DemoOrganizationID, Status: administration.StatusActive, PasswordHash: password.Hash, PasswordSalt: password.Salt, SessionVersion: 1, CreatedAt: now, UpdatedAt: now} +} + +func cloneAsset(a assets.Asset) assets.Asset { + a.Tags = append([]string(nil), a.Tags...) + a.Metadata = cloneMap(a.Metadata) + return a +} +func cloneJob(j jobs.Job) jobs.Job { + j.InputAssetIDs = append([]string(nil), j.InputAssetIDs...) + j.InputURLs = append([]string(nil), j.InputURLs...) + j.OutputAssetIDs = append([]string(nil), j.OutputAssetIDs...) + j.RequestPayload = append(json.RawMessage(nil), j.RequestPayload...) + j.ResponsePayload = append(json.RawMessage(nil), j.ResponsePayload...) + j.WebhookLastStatus = append(json.RawMessage(nil), j.WebhookLastStatus...) + j.UsageContext = append(json.RawMessage(nil), j.UsageContext...) + j.Billing = append(json.RawMessage(nil), j.Billing...) + if j.Error != nil { + e := *j.Error + j.Error = &e + } + if j.ProviderDispatchStartedAt != nil { + value := *j.ProviderDispatchStartedAt + j.ProviderDispatchStartedAt = &value + } + if j.DispatchReadyAt != nil { + value := *j.DispatchReadyAt + j.DispatchReadyAt = &value + } + if j.FinalizedAt != nil { + value := *j.FinalizedAt + j.FinalizedAt = &value + } + return j +} +func cloneMap(m map[string]any) map[string]any { + if m == nil { + return nil + } + raw, err := json.Marshal(m) + if err != nil { + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = v + } + return out + } + var out map[string]any + _ = json.Unmarshal(raw, &out) + return out +} +func cloneRule(r billing.PriceRule) billing.PriceRule { + r.Conditions = billing.Conditions(cloneMap(map[string]any(r.Conditions))) + r.Source = cloneMap(r.Source) + raw, _ := json.Marshal(r.Dimensions) + _ = json.Unmarshal(raw, &r.Dimensions) + return r +} + +// Assets. +func (s *Store) ListOwner(_ context.Context, owner string) ([]assets.Asset, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []assets.Asset{} + for _, a := range s.assets { + if a.OwnerID == owner { + out = append(out, cloneAsset(a)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) }) + return out, nil +} +func (s *Store) GetOwner(_ context.Context, owner, id string) (assets.Asset, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + a, ok := s.assets[id] + if !ok || a.OwnerID != owner { + return assets.Asset{}, false, nil + } + return cloneAsset(a), true, nil +} +func (s *Store) GetOwnerByStoragePath(_ context.Context, owner, path string) (assets.Asset, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + for _, a := range s.assets { + if a.OwnerID == owner && a.StoragePath == path { + return cloneAsset(a), true, nil + } + } + return assets.Asset{}, false, nil +} +func (s *Store) ListPublic(_ context.Context, owner, client string, limit int) ([]assets.Asset, error) { + s.mu.RLock() + defer s.mu.RUnlock() + allowed := s.publicAssetIDs(owner, client, limit) + tag := assets.ClientTag(client) + out := []assets.Asset{} + for _, a := range s.assets { + if a.OwnerID == owner && (contains(a.Tags, tag) || allowed[a.ID]) { + out = append(out, cloneAsset(a)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) }) + return out, nil +} +func (s *Store) GetPublic(_ context.Context, owner, client, id string, limit int) (assets.Asset, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + a, ok := s.assets[id] + if !ok || a.OwnerID != owner { + return assets.Asset{}, false, nil + } + if !contains(a.Tags, assets.ClientTag(client)) && !s.publicAssetIDs(owner, client, limit)[id] { + return assets.Asset{}, false, nil + } + return cloneAsset(a), true, nil +} +func (s *Store) publicAssetIDs(owner, client string, limit int) map[string]bool { + candidates := []jobs.Job{} + for _, j := range s.jobs { + if j.OwnerID == owner && j.ExternalClientID == client { + candidates = append(candidates, j) + } + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].CreatedAt.After(candidates[j].CreatedAt) }) + if limit <= 0 { + limit = 200 + } + if len(candidates) > limit { + candidates = candidates[:limit] + } + out := map[string]bool{} + for _, j := range candidates { + for _, id := range append(append([]string{}, j.InputAssetIDs...), j.OutputAssetIDs...) { + out[id] = true + } + } + return out +} +func (s *Store) Create(_ context.Context, a assets.Asset) (assets.Asset, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.assets[a.ID]; ok { + return assets.Asset{}, fmt.Errorf("asset already exists: %s", a.ID) + } + if a.Tags == nil { + a.Tags = []string{} + } + if a.Metadata == nil { + a.Metadata = map[string]any{} + } + s.assets[a.ID] = cloneAsset(a) + return cloneAsset(a), nil +} +func (s *Store) DeleteOwner(_ context.Context, owner, id string) (assets.Asset, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + a, ok := s.assets[id] + if !ok || a.OwnerID != owner { + return assets.Asset{}, false, nil + } + delete(s.assets, id) + return cloneAsset(a), true, nil +} + +// Jobs. +func (s *Store) ListJobs(_ context.Context, f jobs.ListFilter) ([]jobs.Job, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := []jobs.Job{} + for _, j := range s.jobs { + if !f.Scope.Owns(j) || f.Status != "" && j.Status != f.Status || f.Capability != "" && j.Capability != f.Capability || f.Before != nil && !j.CreatedAt.Before(*f.Before) { + continue + } + out = append(out, cloneJob(j)) + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) }) + if f.Limit > 0 && len(out) > f.Limit { + out = out[:f.Limit] + } + return out, nil +} +func (s *Store) FindJob(_ context.Context, scope jobs.Scope, id string) (jobs.Job, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + j, ok := s.jobs[id] + if !ok || !scope.Owns(j) { + return jobs.Job{}, false, nil + } + return cloneJob(j), true, nil +} +func (s *Store) FindIdempotentJob(_ context.Context, scope jobs.Scope, key string) (jobs.Job, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + for _, j := range s.jobs { + if scope.Owns(j) && j.IdempotencyKey == key { + return cloneJob(j), true, nil + } + } + return jobs.Job{}, false, nil +} +func (s *Store) CreateJob(_ context.Context, j jobs.Job) (jobs.Job, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.jobs[j.ID]; ok { + return jobs.Job{}, fmt.Errorf("generation job already exists: %s", j.ID) + } + if j.ExternalClientID != "" && j.IdempotencyKey != "" { + for _, v := range s.jobs { + if v.OwnerID == j.OwnerID && v.ExternalClientID == j.ExternalClientID && v.IdempotencyKey == j.IdempotencyKey { + return jobs.Job{}, jobs.ErrUniqueIdempotency + } + } + } + s.jobs[j.ID] = cloneJob(j) + return cloneJob(j), nil +} +func (s *Store) UpdateJob(_ context.Context, id string, p jobs.Patch) (jobs.Job, error) { + s.mu.Lock() + defer s.mu.Unlock() + j, ok := s.jobs[id] + if !ok { + return jobs.Job{}, fmt.Errorf("generation job not found: %s", id) + } + if len(p.ExpectedStatuses) > 0 && !containsStatus(p.ExpectedStatuses, j.Status) { + return jobs.Job{}, jobs.ErrTransitionConflict + } + if p.ExpectedLockedBy != nil && j.LockedBy != *p.ExpectedLockedBy { + return jobs.Job{}, jobs.ErrTransitionConflict + } + if p.Status != nil { + j.Status = *p.Status + } + if p.Error != nil { + e := *p.Error + j.Error = &e + } + if p.ClearError { + j.Error = nil + } + if p.Attempts != nil { + j.Attempts = *p.Attempts + } + if p.ScheduledAt != nil { + j.ScheduledAt = *p.ScheduledAt + } + if p.CompletedAt != nil { + v := *p.CompletedAt + j.CompletedAt = &v + } + if p.ProviderTaskID != nil { + j.ProviderTaskID = *p.ProviderTaskID + } + if p.ProviderDispatchStartedAt != nil { + started := *p.ProviderDispatchStartedAt + j.ProviderDispatchStartedAt = &started + } + if p.ClearProviderDispatch { + j.ProviderDispatchStartedAt = nil + } + if p.SetResponsePayload { + j.ResponsePayload = append([]byte(nil), p.ResponsePayload...) + } + if p.DispatchReadyAt != nil { + ready := *p.DispatchReadyAt + j.DispatchReadyAt = &ready + } + if p.FinalizedAt != nil { + finalized := *p.FinalizedAt + j.FinalizedAt = &finalized + } + if p.ClearProviderTaskID { + j.ProviderTaskID = "" + } + if p.ClearLease { + j.LockedAt = nil + j.LockedBy = "" + } + if p.WebhookAttempts != nil { + j.WebhookAttempts = *p.WebhookAttempts + } + if p.SetWebhookStatus { + j.WebhookLastStatus = append([]byte(nil), p.WebhookLastStatus...) + } + j.UpdatedAt = s.now().UTC() + s.jobs[id] = cloneJob(j) + return cloneJob(j), nil +} +func (s *Store) DeleteJob(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.jobs[id]; !ok { + return fmt.Errorf("generation job not found: %s", id) + } + delete(s.jobs, id) + return nil +} +func (s *Store) ClaimJobs(_ context.Context, worker string, limit, timeoutSeconds int) ([]jobs.Job, error) { + s.mu.Lock() + defer s.mu.Unlock() + if limit < 1 { + limit = 1 + } + if limit > 20 { + limit = 20 + } + if timeoutSeconds <= 0 { + timeoutSeconds = 300 + } + now := s.now().UTC() + eligible := []jobs.Job{} + for _, j := range s.jobs { + if j.DispatchReadyAt == nil || j.FinalizedAt != nil || j.ScheduledAt.After(now) { + continue + } + leaseAvailable := j.LockedAt == nil || j.LockedAt.Add(time.Duration(timeoutSeconds)*time.Second).Before(now) + if leaseAvailable && (j.Status == jobs.StatusQueued || j.Status == jobs.StatusRunning || j.Status.Terminal()) { + eligible = append(eligible, j) + } + } + sort.Slice(eligible, func(i, j int) bool { + if eligible[i].Priority != eligible[j].Priority { + return eligible[i].Priority > eligible[j].Priority + } + if !eligible[i].ScheduledAt.Equal(eligible[j].ScheduledAt) { + return eligible[i].ScheduledAt.Before(eligible[j].ScheduledAt) + } + return eligible[i].CreatedAt.Before(eligible[j].CreatedAt) + }) + if len(eligible) > limit { + eligible = eligible[:limit] + } + out := make([]jobs.Job, 0, len(eligible)) + for _, j := range eligible { + j.LockedBy = worker + j.LockedAt = timePtr(now) + if j.StartedAt == nil { + j.StartedAt = timePtr(now) + } + j.UpdatedAt = now + s.jobs[j.ID] = cloneJob(j) + out = append(out, cloneJob(j)) + } + return out, nil +} +func (s *Store) WriteBilling(_ context.Context, id string, value json.RawMessage) error { + if len(value) == 0 || !json.Valid(value) { + return fmt.Errorf("write generation billing: invalid JSON") + } + s.mu.Lock() + defer s.mu.Unlock() + j, ok := s.jobs[id] + if !ok { + return fmt.Errorf("write generation billing: generation job not found") + } + j.Billing = append([]byte(nil), value...) + j.UpdatedAt = s.now().UTC() + s.jobs[id] = j + return nil +} + +func (s *Store) WriteBillingFenced(_ context.Context, id string, value json.RawMessage, status jobs.Status, lockedBy string) error { + if len(value) == 0 || !json.Valid(value) { + return fmt.Errorf("write generation billing: invalid JSON") + } + s.mu.Lock() + defer s.mu.Unlock() + j, ok := s.jobs[id] + if !ok || j.Status != status || j.LockedBy != lockedBy { + return jobs.ErrTransitionConflict + } + j.Billing = append([]byte(nil), value...) + j.UpdatedAt = s.now().UTC() + s.jobs[id] = cloneJob(j) + return nil +} + +func (s *Store) ActivateCreation(_ context.Context, id string, value json.RawMessage) error { + if len(value) == 0 || !json.Valid(value) { + return fmt.Errorf("activate generation creation: invalid JSON") + } + s.mu.Lock() + defer s.mu.Unlock() + j, ok := s.jobs[id] + if !ok { + return fmt.Errorf("activate generation creation: generation job not found") + } + if j.DispatchReadyAt != nil { + if !jsonEqual(j.Billing, value) { + return jobs.ErrTransitionConflict + } + return nil + } + j.Billing = append([]byte(nil), value...) + ready := s.now().UTC() + j.DispatchReadyAt = &ready + j.UpdatedAt = s.now().UTC() + s.jobs[id] = cloneJob(j) + return nil +} + +func jsonEqual(a, b json.RawMessage) bool { + var left, right any + return json.Unmarshal(a, &left) == nil && json.Unmarshal(b, &right) == nil && reflect.DeepEqual(left, right) +} +func (s *Store) WriteOutputAssetIDs(_ context.Context, id string, values []string) error { + s.mu.Lock() + defer s.mu.Unlock() + j, ok := s.jobs[id] + if !ok { + return fmt.Errorf("write generation output assets: generation job not found") + } + j.OutputAssetIDs = append([]string{}, values...) + j.UpdatedAt = s.now().UTC() + s.jobs[id] = j + return nil +} + +func (s *Store) WriteOutputAssetIDsFenced(_ context.Context, id string, values []string, status jobs.Status, lockedBy string) error { + s.mu.Lock() + defer s.mu.Unlock() + j, ok := s.jobs[id] + if !ok || j.Status != status || j.LockedBy != lockedBy { + return jobs.ErrTransitionConflict + } + j.OutputAssetIDs = append([]string{}, values...) + j.UpdatedAt = s.now().UTC() + s.jobs[id] = cloneJob(j) + return nil +} +func (s *Store) FailCreation(_ context.Context, j jobs.Job) error { + if j.Status != jobs.StatusFailed || j.Error == nil || len(j.Billing) == 0 || !json.Valid(j.Billing) { + return fmt.Errorf("fail generation creation: invalid terminal state") + } + s.mu.Lock() + defer s.mu.Unlock() + current, ok := s.jobs[j.ID] + if !ok { + return fmt.Errorf("fail generation creation: generation job not found") + } + current.Status = j.Status + e := *j.Error + current.Error = &e + current.Billing = append([]byte(nil), j.Billing...) + now := s.now().UTC() + if current.CompletedAt == nil { + current.CompletedAt = &now + } + current.LockedAt = nil + current.LockedBy = "" + current.UpdatedAt = now + s.jobs[j.ID] = current + return nil +} + +func timePtr(v time.Time) *time.Time { return &v } +func contains(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + return false +} + +func containsStatus(values []jobs.Status, want jobs.Status) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/backend/internal/localstore/store_test.go b/backend/internal/localstore/store_test.go new file mode 100644 index 0000000..5a2d7b2 --- /dev/null +++ b/backend/internal/localstore/store_test.go @@ -0,0 +1,308 @@ +package localstore_test + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/localstore" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage" +) + +var fixed = time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + +func TestStoreSatisfiesApplicationPersistenceSeams(t *testing.T) { + store := localstore.New(localstore.WithClock(func() time.Time { return fixed })) + var _ administration.Store = store + var _ assets.Catalog = store + var _ billing.Store = store + var _ billing.PriceRuleSeeder = store + var _ billing.WalletPoster = store + var _ jobs.Store = store + var _ templates.Catalog = store + var _ usage.Repository = store + var _ identity.AuthorizationSnapshotLoader = store + var _ identity.CredentialAuthenticator = store + var _ identity.PasswordChanger = store + var _ interface { + GetOwnerByStoragePath(context.Context, string, string) (assets.Asset, bool, error) + FailCreation(context.Context, jobs.Job) error + WriteBilling(context.Context, string, json.RawMessage) error + WriteOutputAssetIDs(context.Context, string, []string) error + } = store +} + +func TestAssetLifecycleIsOwnerScopedAndPublicAccessUsesTagsOrJobs(t *testing.T) { + ctx := context.Background() + store := localstore.New() + a := assets.Asset{ID: "asset-1", OwnerID: "owner-a", StoragePath: "uploads/a.png", Tags: []string{assets.ClientTag("client-a")}, Metadata: map[string]any{"x": "y"}, CreatedAt: fixed} + if _, err := store.Create(ctx, a); err != nil { + t.Fatal(err) + } + if _, found, _ := store.GetOwner(ctx, "owner-b", a.ID); found { + t.Fatal("cross-owner asset leaked") + } + if got, found, _ := store.GetOwnerByStoragePath(ctx, a.OwnerID, a.StoragePath); !found || got.ID != a.ID { + t.Fatalf("storage lookup = %#v, %v", got, found) + } + if got, _ := store.ListPublic(ctx, a.OwnerID, "client-a", 10); len(got) != 1 { + t.Fatalf("public tagged assets = %d", len(got)) + } + if _, found, _ := store.DeleteOwner(ctx, "owner-b", a.ID); found { + t.Fatal("cross-owner delete succeeded") + } +} + +func TestJobClaimsRespectScheduleLeaseAndOrchestrationWrites(t *testing.T) { + ctx := context.Background() + store := localstore.New(localstore.WithClock(func() time.Time { return fixed })) + ready := fixed.Add(-time.Minute) + job := jobs.Job{ID: "job-1", OwnerID: "owner", ExternalClientID: "client", Status: jobs.StatusQueued, Capability: "image.generate", Provider: "mock", ReqKey: "mock", ScheduledAt: fixed.Add(-time.Minute), DispatchReadyAt: &ready, CreatedAt: fixed.Add(-time.Hour), UpdatedAt: fixed.Add(-time.Hour)} + if _, err := store.CreateJob(ctx, job); err != nil { + t.Fatal(err) + } + claimed, err := store.ClaimJobs(ctx, "worker-a", 1, 300) + if err != nil || len(claimed) != 1 || claimed[0].Status != jobs.StatusQueued || claimed[0].LockedBy != "worker-a" || claimed[0].Attempts != 0 { + t.Fatalf("claim = %#v, %v", claimed, err) + } + if again, _ := store.ClaimJobs(ctx, "worker-b", 1, 300); len(again) != 0 { + t.Fatal("active lease was reclaimed") + } + if err := store.WriteBilling(ctx, job.ID, json.RawMessage(`{"amountFen":12}`)); err != nil { + t.Fatal(err) + } + if err := store.WriteOutputAssetIDs(ctx, job.ID, []string{"asset-out"}); err != nil { + t.Fatal(err) + } + failed := job + failed.Status = jobs.StatusFailed + failed.Error = &jobs.JobError{Message: "charge failed"} + failed.Billing = json.RawMessage(`{"status":"failed"}`) + if err := store.FailCreation(ctx, failed); err != nil { + t.Fatal(err) + } + got, found, _ := store.FindJob(ctx, jobs.Scope{OwnerID: "owner"}, job.ID) + if !found || got.Status != jobs.StatusFailed || got.LockedBy != "" || got.OutputAssetIDs[0] != "asset-out" { + t.Fatalf("job = %#v", got) + } +} + +func TestWalletPostingIsIdempotentAndNeverOverdrafts(t *testing.T) { + ctx := context.Background() + store := localstore.New(localstore.WithClock(func() time.Time { return fixed })) + org := administration.Organization{ID: "org", Name: "Org", Status: administration.StatusActive, CreatedAt: fixed, UpdatedAt: fixed} + if _, err := store.CreateOrganization(ctx, org); err != nil { + t.Fatal(err) + } + credit := billing.WalletPostParams{LedgerID: "l1", OrganizationID: org.ID, Kind: "recharge", DeltaFen: 100, Currency: billing.CurrencyCNY, IdempotencyKey: "credit-1"} + first, err := store.PostWalletEntry(ctx, credit) + if err != nil { + t.Fatal(err) + } + repeated, err := store.PostWalletEntry(ctx, credit) + if err != nil || repeated.LedgerID != first.LedgerID { + t.Fatalf("repeat = %#v, %v", repeated, err) + } + changed := credit + changed.DeltaFen = 99 + if _, err := store.PostWalletEntry(ctx, changed); err == nil { + t.Fatal("payload mismatch accepted") + } + if _, err := store.PostWalletEntry(ctx, billing.WalletPostParams{LedgerID: "l2", OrganizationID: org.ID, Kind: "charge", DeltaFen: -101, Currency: billing.CurrencyCNY, IdempotencyKey: "charge-1"}); err == nil { + t.Fatal("overdraft accepted") + } + wallet, _ := store.BillingWallet(ctx, org.ID) + if wallet.BalanceFen != 100 || wallet.TotalRechargedFen != 100 { + t.Fatalf("wallet = %#v", wallet) + } +} + +func TestWalletPostingIdempotencyIgnoresDescriptionMetadataAndLedgerID(t *testing.T) { + ctx := context.Background() + store := localstore.New(localstore.WithClock(func() time.Time { return fixed })) + params := billing.WalletPostParams{ + LedgerID: "ledger-original", OrganizationID: "org", AccountID: "account", JobID: "job", + Kind: "recharge", DeltaFen: 100, Currency: billing.CurrencyCNY, IdempotencyKey: "credit", + Description: "original", Metadata: map[string]any{"request": "original"}, + } + first, err := store.PostWalletEntry(ctx, params) + if err != nil { + t.Fatal(err) + } + replay := params + replay.LedgerID = "ledger-replay" + replay.Description = "changed" + replay.Metadata = map[string]any{"request": "changed", "extra": true} + got, err := store.PostWalletEntry(ctx, replay) + if err != nil { + t.Fatalf("replay with non-conflicting fields: %v", err) + } + if got.LedgerID != first.LedgerID { + t.Fatalf("replay ledger = %q, want original %q", got.LedgerID, first.LedgerID) + } + ledger, err := store.BillingLedger(ctx, "org", "", 10) + if err != nil || len(ledger) != 1 { + t.Fatalf("ledger = %#v, %v", ledger, err) + } +} + +func TestActivateCreationReplayRequiresSemanticallyEqualBilling(t *testing.T) { + ctx := context.Background() + store := localstore.New(localstore.WithClock(func() time.Time { return fixed })) + job := jobs.Job{ID: "activation", OwnerID: "owner", Status: jobs.StatusQueued, ScheduledAt: fixed, CreatedAt: fixed, UpdatedAt: fixed} + if _, err := store.CreateJob(ctx, job); err != nil { + t.Fatal(err) + } + if err := store.ActivateCreation(ctx, job.ID, json.RawMessage(`{"status":"charged","details":{"amountFen":12,"currency":"CNY"}}`)); err != nil { + t.Fatal(err) + } + if err := store.ActivateCreation(ctx, job.ID, json.RawMessage(`{ "details": { "currency": "CNY", "amountFen": 12 }, "status": "charged" }`)); err != nil { + t.Fatalf("semantic replay: %v", err) + } + if err := store.ActivateCreation(ctx, job.ID, json.RawMessage(`{"status":"charged","details":{"amountFen":13,"currency":"CNY"}}`)); !errors.Is(err, jobs.ErrTransitionConflict) { + t.Fatalf("different replay error = %v", err) + } + got, found, err := store.FindJob(ctx, jobs.Scope{OwnerID: job.OwnerID}, job.ID) + if err != nil || !found || string(got.Billing) != `{"status":"charged","details":{"amountFen":12,"currency":"CNY"}}` { + t.Fatalf("job after conflict = %#v, %v, %v", got, found, err) + } +} + +func TestDeleteAccountArchivesUsageEvents(t *testing.T) { + ctx := context.Background() + store := localstore.New(localstore.WithClock(func() time.Time { return fixed })) + account := administration.Account{ID: "delete-me", Phone: "13900139000", OrganizationID: localstore.DemoOrganizationID, Status: administration.StatusActive, CreatedAt: fixed, UpdatedAt: fixed} + if _, err := store.CreateAccount(ctx, account); err != nil { + t.Fatal(err) + } + event := usage.Event{ID: "usage-delete", JobID: "job-delete", OwnerID: account.ID, OrganizationID: account.OrganizationID, CreatedAt: fixed.Format(time.RFC3339)} + if _, inserted, err := store.Insert(event); err != nil || !inserted { + t.Fatalf("insert usage = %v, %v", inserted, err) + } + if err := store.DeleteAccount(ctx, account.ID, "archive-owner"); err != nil { + t.Fatal(err) + } + archived, err := store.List(usage.Filters{OwnerID: "archive-owner"}) + if err != nil || len(archived) != 1 || archived[0].ID != event.ID { + t.Fatalf("archived usage = %#v, %v", archived, err) + } + original, err := store.List(usage.Filters{OwnerID: account.ID}) + if err != nil || len(original) != 0 { + t.Fatalf("original usage = %#v, %v", original, err) + } +} + +func TestGenerationLifecycleRejectsStaleCASAndFencedWriters(t *testing.T) { + ctx := context.Background() + now := fixed + store := localstore.New(localstore.WithClock(func() time.Time { return now })) + ready := fixed + job := jobs.Job{ID: "fenced", OwnerID: "owner", Status: jobs.StatusQueued, ScheduledAt: fixed, DispatchReadyAt: &ready, CreatedAt: fixed, UpdatedAt: fixed} + if _, err := store.CreateJob(ctx, job); err != nil { + t.Fatal(err) + } + first, err := store.ClaimJobs(ctx, "worker-old", 1, 60) + if err != nil || len(first) != 1 { + t.Fatalf("first claim = %#v, %v", first, err) + } + now = fixed.Add(2 * time.Minute) + second, err := store.ClaimJobs(ctx, "worker-new", 1, 60) + if err != nil || len(second) != 1 || second[0].LockedBy != "worker-new" { + t.Fatalf("second claim = %#v, %v", second, err) + } + oldWorker := "worker-old" + running := jobs.StatusRunning + if _, err := store.UpdateJob(ctx, job.ID, jobs.Patch{Status: &running, ExpectedStatuses: []jobs.Status{jobs.StatusQueued}, ExpectedLockedBy: &oldWorker}); !errors.Is(err, jobs.ErrTransitionConflict) { + t.Fatalf("stale CAS error = %v", err) + } + if err := store.WriteBillingFenced(ctx, job.ID, json.RawMessage(`{"status":"old"}`), jobs.StatusQueued, oldWorker); !errors.Is(err, jobs.ErrTransitionConflict) { + t.Fatalf("stale billing writer error = %v", err) + } + if err := store.WriteOutputAssetIDsFenced(ctx, job.ID, []string{"old-output"}, jobs.StatusQueued, oldWorker); !errors.Is(err, jobs.ErrTransitionConflict) { + t.Fatalf("stale output writer error = %v", err) + } + newWorker := "worker-new" + updated, err := store.UpdateJob(ctx, job.ID, jobs.Patch{Status: &running, ExpectedStatuses: []jobs.Status{jobs.StatusQueued}, ExpectedLockedBy: &newWorker}) + if err != nil || updated.Status != jobs.StatusRunning { + t.Fatalf("current CAS update = %#v, %v", updated, err) + } + if err := store.WriteBillingFenced(ctx, job.ID, json.RawMessage(`{"status":"current"}`), jobs.StatusRunning, newWorker); err != nil { + t.Fatalf("current billing writer: %v", err) + } + if err := store.WriteOutputAssetIDsFenced(ctx, job.ID, []string{"current-output"}, jobs.StatusRunning, newWorker); err != nil { + t.Fatalf("current output writer: %v", err) + } + got, found, err := store.FindJob(ctx, jobs.Scope{OwnerID: job.OwnerID}, job.ID) + if err != nil || !found || got.LockedBy != newWorker || got.Status != jobs.StatusRunning || string(got.Billing) != `{"status":"current"}` || len(got.OutputAssetIDs) != 1 || got.OutputAssetIDs[0] != "current-output" { + t.Fatalf("job after stale writes = %#v, %v, %v", got, found, err) + } +} + +func TestAdministrationIdentityAndPasswordLifecycleShareState(t *testing.T) { + ctx := context.Background() + store := localstore.New(localstore.WithClock(func() time.Time { return fixed })) + seed, found, err := store.GetAccount(ctx, localstore.DemoAccountID) + if err != nil || !found || seed.Role != administration.RoleSuperAdmin || seed.OrganizationID != localstore.DemoOrganizationID { + t.Fatalf("demo = %#v, %v, %v", seed, found, err) + } + hashed, _ := administration.HashPassword("old-password") + account := administration.Account{ID: "user-1", Phone: "13800138000", DisplayName: "User", Role: administration.RoleUser, OrganizationID: localstore.DemoOrganizationID, Status: administration.StatusActive, PasswordHash: hashed.Hash, PasswordSalt: hashed.Salt, SessionVersion: 1, CreatedAt: fixed, UpdatedAt: fixed} + if _, err := store.CreateAccount(ctx, account); err != nil { + t.Fatal(err) + } + if _, err := store.AttemptPasswordLogin(ctx, account.Phone, "wrong", fixed); !errors.Is(err, identity.ErrPasswordLogin) { + t.Fatalf("wrong password error = %v", err) + } + login, err := store.AttemptPasswordLogin(ctx, account.Phone, "old-password", fixed) + if err != nil || login.Account.ID != account.ID { + t.Fatalf("login = %#v, %v", login, err) + } + snapshot, err := store.ChangeOwnPassword(ctx, account.ID, "old-password", "new-password", fixed.Add(time.Hour)) + if err != nil || snapshot.Account.SessionVersion != 2 { + t.Fatalf("change = %#v, %v", snapshot, err) + } + if _, err := store.AttemptPasswordLogin(ctx, account.Phone, "old-password", fixed.Add(time.Hour)); !errors.Is(err, identity.ErrPasswordLogin) { + t.Fatal("old password still works") + } + if _, err := store.AttemptPasswordLogin(ctx, account.Phone, "new-password", fixed.Add(time.Hour)); err != nil { + t.Fatal(err) + } +} + +func TestTemplatesUsageAndPriceSeedingAreIdempotentAndScoped(t *testing.T) { + ctx := context.Background() + store := localstore.New(localstore.WithClock(func() time.Time { return fixed })) + template := templates.Template{ID: "tpl", OwnerID: "owner", Name: "T", Prompt: "P", CreatedAt: fixed, UpdatedAt: fixed} + if _, err := store.CreateTemplate(ctx, template); err != nil { + t.Fatal(err) + } + if items, _ := store.ListTemplates(ctx, "other"); len(items) != 0 { + t.Fatal("template leaked") + } + event := usage.Event{ID: "usage-1", JobID: "job-1", OwnerID: "owner", Capability: "image.generate", CreatedAt: fixed.Format(time.RFC3339)} + if _, inserted, _ := store.Insert(event); !inserted { + t.Fatal("first usage not inserted") + } + if _, inserted, _ := store.Insert(event); inserted { + t.Fatal("duplicate job usage inserted") + } + rules := billing.DefaultBillingPriceRules()[:1] + if err := store.SeedBillingPriceRules(ctx, rules); err != nil { + t.Fatal(err) + } + if err := store.SeedBillingPriceRules(ctx, rules); err != nil { + t.Fatal(err) + } + got, _ := store.ListBillingPriceRules(ctx, true) + if len(got) != 1 { + t.Fatalf("rules = %d", len(got)) + } +} diff --git a/backend/internal/logging/service.go b/backend/internal/logging/service.go new file mode 100644 index 0000000..c1f4cb8 --- /dev/null +++ b/backend/internal/logging/service.go @@ -0,0 +1,318 @@ +package logging + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + "sync" + "time" +) + +type Level string + +const ( + Info Level = "info" + Warning Level = "warning" + Error Level = "error" +) + +type Entry struct { + ID string `json:"id"` + CreatedAt string `json:"createdAt"` + Level Level `json:"level"` + Source string `json:"source"` + Message string `json:"message"` + Status int `json:"status,omitempty"` + Method string `json:"method,omitempty"` + Path string `json:"path,omitempty"` + Stack string `json:"stack,omitempty"` + Details any `json:"details,omitempty"` +} +type Input struct { + Level Level + Source string + Message string + Error error + Status int + Method string + Path string + Stack string + Details any +} +type Filters struct { + Level, Q, Source string + Limit int +} +type Service struct { + mu sync.Mutex + path string + maxBytes int64 + now func() time.Time + newID func() string +} + +func New(path string, maxBytes int64, now func() time.Time, newID func() string) *Service { + if maxBytes <= 0 { + maxBytes = 5 * 1024 * 1024 + } + if now == nil { + now = time.Now + } + if newID == nil { + newID = randomID + } + return &Service{path: path, maxBytes: maxBytes, now: now, newID: newID} +} +func (s *Service) Append(ctx context.Context, input Input) (Entry, error) { + if err := ctx.Err(); err != nil { + return Entry{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + level := input.Level + if !validLevel(level) { + if input.Status >= 500 { + level = Error + } else { + level = Info + } + } + message := input.Message + if message == "" && input.Error != nil { + message = input.Error.Error() + } + if message == "" { + message = "Unknown log event" + } + stack := input.Stack + if stack == "" && input.Error != nil { + stack = input.Error.Error() + } + entry := Entry{ID: "log_" + s.newID(), CreatedAt: s.now().UTC().Format(time.RFC3339Nano), Level: level, Source: sanitizeText(defaultString(input.Source, "server")), Message: sanitizeText(message), Status: input.Status, Method: sanitizeText(input.Method), Path: sanitizeText(input.Path), Stack: sanitizeText(stack), Details: sanitize(input.Details, 0, map[visit]bool{})} + line, err := json.Marshal(entry) + if err != nil { + return Entry{}, err + } + if err = os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil { + return Entry{}, err + } + if info, statErr := os.Stat(s.path); statErr == nil && info.Size() >= s.maxBytes { + if err = os.Rename(s.path, s.path+".1"); err != nil { + return Entry{}, err + } + } else if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + return Entry{}, statErr + } + file, err := os.OpenFile(s.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return Entry{}, err + } + _, writeErr := file.Write(append(line, '\n')) + closeErr := file.Close() + if writeErr != nil { + return Entry{}, writeErr + } + if closeErr != nil { + return Entry{}, closeErr + } + return entry, nil +} +func (s *Service) List(ctx context.Context, filters Filters) ([]Entry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + s.mu.Lock() + defer s.mu.Unlock() + data, err := os.ReadFile(s.path) + if errors.Is(err, os.ErrNotExist) { + return []Entry{}, nil + } + if err != nil { + return nil, err + } + limit := filters.Limit + if limit == 0 { + limit = 100 + } + if limit < 1 { + limit = 1 + } + if limit > 500 { + limit = 500 + } + q := strings.ToLower(strings.TrimSpace(filters.Q)) + source := strings.ToLower(strings.TrimSpace(filters.Source)) + level := strings.TrimSpace(filters.Level) + lines := strings.Split(string(data), "\n") + result := make([]Entry, 0, limit) + for i := len(lines) - 1; i >= 0 && len(result) < limit; i-- { + if strings.TrimSpace(lines[i]) == "" { + continue + } + var entry Entry + if json.Unmarshal([]byte(lines[i]), &entry) != nil || entry.ID == "" || entry.CreatedAt == "" || entry.Source == "" || entry.Message == "" || !validLevel(entry.Level) { + continue + } + if level != "" && level != "all" && string(entry.Level) != level { + continue + } + if source != "" && !strings.Contains(strings.ToLower(entry.Source), source) { + continue + } + if q != "" && !strings.Contains(strings.ToLower(search(entry)), q) { + continue + } + result = append(result, entry) + } + return result, nil +} +func (s *Service) Clear(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil { + return err + } + return os.WriteFile(s.path, []byte{}, 0o600) +} + +const maxText = 20000 + +var sensitiveKey = regexp.MustCompile(`(?i)api[_-]?key|access[_-]?key|secret|token|password|authorization|credential`) +var authorizationText = regexp.MustCompile(`(?i)(authorization\s*[:=]\s*)[^\s,;}]+`) +var authorizationBearerText = regexp.MustCompile(`(?i)(authorization\s*[:=]\s*bearer\s+)[^\s,;}]+`) +var bearerText = regexp.MustCompile(`(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+`) +var sensitiveText = regexp.MustCompile(`(?i)((?:api[_-]?key|access[_-]?key|secret|token|password|client[_-]?secret)\s*[:=]\s*)[^\s,;}]+`) +var jwtText = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b`) + +func sanitizeText(input string) string { + input = truncate(input) + input = authorizationBearerText.ReplaceAllString(input, "${1}[redacted]") + input = authorizationText.ReplaceAllString(input, "${1}[redacted]") + input = bearerText.ReplaceAllString(input, "${1}[redacted]") + input = sensitiveText.ReplaceAllString(input, "${1}[redacted]") + return jwtText.ReplaceAllString(input, "[jwt-redacted]") +} +func truncate(input string) string { + runes := []rune(input) + if len(runes) <= maxText { + return input + } + return string(runes[:maxText]) + "...[truncated]" +} + +type visit struct { + kind reflect.Kind + pointer uintptr +} + +func sanitize(value any, depth int, seen map[visit]bool) any { + if value == nil { + return nil + } + if text, ok := value.(string); ok { + return sanitizeText(text) + } + v := reflect.ValueOf(value) + for v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer { + if v.IsNil() { + return nil + } + key := visit{v.Kind(), v.Pointer()} + if seen[key] { + return "[circular]" + } + seen[key] = true + v = v.Elem() + } + if depth >= 5 { + return "[truncated]" + } + switch v.Kind() { + case reflect.Bool: + return v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return v.Uint() + case reflect.Float32, reflect.Float64: + return v.Float() + case reflect.String: + return sanitizeText(v.String()) + case reflect.Slice, reflect.Array: + length := v.Len() + if length > 50 { + length = 50 + } + out := make([]any, length) + for i := 0; i < length; i++ { + out[i] = sanitize(v.Index(i).Interface(), depth+1, seen) + } + return out + case reflect.Map: + if v.Type().Key().Kind() != reflect.String { + return sanitizeText(fmt.Sprint(value)) + } + keys := v.MapKeys() + if len(keys) > 80 { + keys = keys[:80] + } + out := map[string]any{} + for _, key := range keys { + name := key.String() + if sensitiveKey.MatchString(name) { + out[name] = "[redacted]" + } else { + out[name] = sanitize(v.MapIndex(key).Interface(), depth+1, seen) + } + } + return out + case reflect.Struct: + out := map[string]any{} + typeOfValue := v.Type() + for index := 0; index < v.NumField() && index < 80; index++ { + fieldValue := v.Field(index) + if !fieldValue.CanInterface() { + continue + } + name := typeOfValue.Field(index).Name + if sensitiveKey.MatchString(name) { + out[name] = "[redacted]" + } else { + out[name] = sanitize(fieldValue.Interface(), depth+1, seen) + } + } + return out + default: + return sanitizeText(fmt.Sprint(value)) + } +} +func validLevel(level Level) bool { return level == Info || level == Warning || level == Error } +func search(entry Entry) string { + details, _ := json.Marshal(entry.Details) + return strings.Join([]string{entry.Message, entry.Source, entry.Path, entry.Method, strconv.Itoa(entry.Status), entry.Stack, string(details)}, " ") +} +func defaultString(value, fallback string) string { + if value == "" { + return fallback + } + return value +} +func randomID() string { + buffer := make([]byte, 9) + if _, err := rand.Read(buffer); err != nil { + return strconv.FormatInt(time.Now().UnixNano(), 36) + } + return hex.EncodeToString(buffer) +} diff --git a/backend/internal/logging/service_test.go b/backend/internal/logging/service_test.go new file mode 100644 index 0000000..109aee8 --- /dev/null +++ b/backend/internal/logging/service_test.go @@ -0,0 +1,123 @@ +package logging + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestFixtureIsConsumedByGo(t *testing.T) { + data, err := os.ReadFile("../../../contracts/logs/runtime-v1.json") + if err != nil { + t.Fatal(err) + } + var fixture struct { + Version, DefaultLimit, MinimumLimit, MaximumLimit, MaximumTextLength, MaximumDepth int + FileName, RotatedSuffix string + } + if err := json.Unmarshal(data, &fixture); err != nil { + t.Fatal(err) + } + if fixture.Version != 1 || fixture.FileName != "server-events.jsonl" || fixture.MaximumLimit != 500 || fixture.MaximumDepth != 5 { + t.Fatalf("fixture = %#v", fixture) + } +} + +func TestServiceAppendsListsFiltersAndClears(t *testing.T) { + path := filepath.Join(t.TempDir(), "server-events.jsonl") + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + service := New(path, 1<<20, func() time.Time { now = now.Add(time.Second); return now }, func() string { return "fixed-id" }) + first, err := service.Append(context.Background(), Input{Level: Error, Source: "api.test", Message: "Authorization: Bearer top-secret", Status: 500, Method: "POST", Path: "/api/test?token=abc", Stack: "password=hunter2", Details: map[string]any{"Authorization": "Bearer hidden", "nested": map[string]any{"api_key": "key", "prompt": "EvoLink"}}}) + if err != nil { + t.Fatal(err) + } + if first.ID != "log_fixed-id" { + t.Fatalf("entry = %#v", first) + } + if _, err := service.Append(context.Background(), Input{Level: Warning, Source: "worker.test", Message: "retry scheduled"}); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, append(mustRead(t, path), []byte("not-json\n")...), 0o600); err != nil { + t.Fatal(err) + } + entries, err := service.List(context.Background(), Filters{Level: "error", Q: "evolink", Source: "API", Limit: 900}) + if err != nil || len(entries) != 1 || entries[0].Source != "api.test" { + t.Fatalf("entries = %#v, err = %v", entries, err) + } + raw := string(mustRead(t, path)) + for _, secret := range []string{"top-secret", "abc", "hunter2", "hidden", "\"key\""} { + if strings.Contains(raw, secret) { + t.Fatalf("raw log leaked %q: %s", secret, raw) + } + } + all, err := service.List(context.Background(), Filters{}) + if err != nil || len(all) != 2 || all[0].Message != "retry scheduled" { + t.Fatalf("all = %#v, err = %v", all, err) + } + if err := service.Clear(context.Background()); err != nil { + t.Fatal(err) + } + if entries, err := service.List(context.Background(), Filters{}); err != nil || len(entries) != 0 { + t.Fatalf("after clear = %#v, %v", entries, err) + } +} + +func TestServiceRotatesAndBoundsNestedDetails(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "server-events.jsonl") + service := New(path, 200, time.Now, func() string { return "id" }) + if _, err := service.Append(context.Background(), Input{Source: "one", Message: strings.Repeat("x", 250)}); err != nil { + t.Fatal(err) + } + if _, err := service.Append(context.Background(), Input{Source: "two", Message: "next", Details: map[string]any{"jwt": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature", "deep": nested(8)}}); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path + ".1"); err != nil { + t.Fatalf("rotated log: %v", err) + } + raw := string(mustRead(t, path)) + if strings.Contains(raw, "eyJhbGci") || !strings.Contains(raw, "[truncated]") { + t.Fatalf("bounds/redaction missing: %s", raw) + } +} + +func TestServiceSerializesConcurrentAppends(t *testing.T) { + path := filepath.Join(t.TempDir(), "server-events.jsonl") + service := New(path, 1<<20, time.Now, nil) + var wait sync.WaitGroup + for index := 0; index < 40; index++ { + wait.Add(1) + go func(index int) { + defer wait.Done() + if _, err := service.Append(context.Background(), Input{Source: "parallel", Message: fmt.Sprintf("entry-%d", index)}); err != nil { + t.Errorf("append %d: %v", index, err) + } + }(index) + } + wait.Wait() + entries, err := service.List(context.Background(), Filters{Limit: 100}) + if err != nil || len(entries) != 40 { + t.Fatalf("entries = %d, err = %v", len(entries), err) + } +} + +func nested(depth int) any { + if depth == 0 { + return "leaf" + } + return map[string]any{"child": nested(depth - 1)} +} +func mustRead(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return data +} diff --git a/backend/internal/orchestration/artifacts.go b/backend/internal/orchestration/artifacts.go new file mode 100644 index 0000000..be30263 --- /dev/null +++ b/backend/internal/orchestration/artifacts.go @@ -0,0 +1,46 @@ +package orchestration + +import ( + "context" + "errors" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" +) + +type ScopedAssetDeleter interface { + Delete(context.Context, assets.Scope, string) (assets.Asset, error) +} + +type AssetArtifacts struct{ assets ScopedAssetDeleter } + +func NewAssetArtifacts(service ScopedAssetDeleter) *AssetArtifacts { + return &AssetArtifacts{assets: service} +} + +// DeleteOutputs is retry-safe: already absent objects are treated as deleted, +// while persistence/storage failures stop job deletion. +func (adapter *AssetArtifacts) DeleteOutputs(ctx context.Context, job jobs.Job) ([]string, error) { + if adapter == nil || adapter.assets == nil || job.OwnerID == "" { + return nil, errors.New("delete generation output assets") + } + seen := make(map[string]bool, len(job.OutputAssetIDs)) + deleted := make([]string, 0, len(job.OutputAssetIDs)) + for _, id := range job.OutputAssetIDs { + if id == "" || seen[id] { + continue + } + seen[id] = true + asset, err := adapter.assets.Delete(ctx, assets.PlatformScope(job.OwnerID), id) + if errors.Is(err, assets.ErrNotFound) { + continue + } + if err != nil { + return nil, errors.New("delete generation output assets") + } + deleted = append(deleted, asset.ID) + } + return deleted, nil +} + +var _ jobs.ArtifactDeleter = (*AssetArtifacts)(nil) diff --git a/backend/internal/orchestration/artifacts_test.go b/backend/internal/orchestration/artifacts_test.go new file mode 100644 index 0000000..2d007f1 --- /dev/null +++ b/backend/internal/orchestration/artifacts_test.go @@ -0,0 +1,51 @@ +package orchestration + +import ( + "context" + "errors" + "reflect" + "testing" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" +) + +func TestAssetArtifactsDeletesUniqueOutputsAndIgnoresAlreadyMissingAssets(t *testing.T) { + deletions := &recordingAssetDeleter{missing: map[string]bool{"asset-missing": true}} + adapter := NewAssetArtifacts(deletions) + + deleted, err := adapter.DeleteOutputs(context.Background(), jobs.Job{ + OwnerID: "account-1", OutputAssetIDs: []string{"asset-1", "asset-missing", "asset-1", "asset-2"}, + }) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(deleted, []string{"asset-1", "asset-2"}) || !reflect.DeepEqual(deletions.ids, []string{"asset-1", "asset-missing", "asset-2"}) { + t.Fatalf("deleted=%#v calls=%#v", deleted, deletions.ids) + } +} + +func TestAssetArtifactsFailsClosedOnStorageError(t *testing.T) { + deletions := &recordingAssetDeleter{failure: errors.New("storage unavailable")} + _, err := NewAssetArtifacts(deletions).DeleteOutputs(context.Background(), jobs.Job{OwnerID: "account-1", OutputAssetIDs: []string{"asset-1"}}) + if err == nil || err.Error() != "delete generation output assets" { + t.Fatalf("err=%v", err) + } +} + +type recordingAssetDeleter struct { + ids []string + missing map[string]bool + failure error +} + +func (d *recordingAssetDeleter) Delete(_ context.Context, _ assets.Scope, id string) (assets.Asset, error) { + d.ids = append(d.ids, id) + if d.failure != nil { + return assets.Asset{}, d.failure + } + if d.missing[id] { + return assets.Asset{}, assets.ErrNotFound + } + return assets.Asset{ID: id}, nil +} diff --git a/backend/internal/orchestration/creation.go b/backend/internal/orchestration/creation.go new file mode 100644 index 0000000..78c2a68 --- /dev/null +++ b/backend/internal/orchestration/creation.go @@ -0,0 +1,253 @@ +package orchestration + +import ( + "context" + "encoding/json" + "errors" + "strings" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/providers" +) + +type CreationInput struct { + OwnerID, ExternalClientID string + Capability string + Body map[string]any + IdempotencyKey string + RetryOf string +} + +type CreationBuilder interface { + Build(context.Context, string, string, string, string, map[string]any) (jobs.CreateCommand, error) +} + +type JobCreator interface { + Create(context.Context, jobs.CreateCommand) (jobs.Job, bool, error) +} + +type BillingQuoter interface { + Quote(context.Context, billing.QuoteCommand) (*billing.Quote, error) +} + +type ChargeLedger interface { + Charge(context.Context, billing.ChargeRequest) (billing.WalletPosting, error) +} + +type CreationStateWriter interface { + WriteBilling(context.Context, string, json.RawMessage) error + ActivateCreation(context.Context, string, json.RawMessage) error + FailCreation(context.Context, jobs.Job) error +} + +type AtomicCreationCharger interface { + ChargeAndActivateCreation(context.Context, billing.ChargeRequest, json.RawMessage) (json.RawMessage, error) +} + +// CreationCoordinator is the single creation transaction policy used by HTTP. +// Its persistence port intentionally exposes durable state transitions rather +// than wallet or SQL details. +type CreationCoordinator struct { + builder CreationBuilder + jobs JobCreator + quotes BillingQuoter + charges ChargeLedger + state CreationStateWriter +} + +func NewCreationCoordinator(builder CreationBuilder, creator JobCreator, quotes BillingQuoter, charges ChargeLedger, state CreationStateWriter) *CreationCoordinator { + return &CreationCoordinator{builder: builder, jobs: creator, quotes: quotes, charges: charges, state: state} +} + +func (c *CreationCoordinator) CreatePlatform(ctx context.Context, session identity.Session, input CreationInput) (jobs.Job, bool, error) { + if c == nil || c.builder == nil || c.jobs == nil || c.quotes == nil || c.charges == nil || c.state == nil { + return jobs.Job{}, false, errors.New("create generation job") + } + input.OwnerID = session.User.ID + input.ExternalClientID = "" + command, err := c.builder.Build(ctx, input.OwnerID, "", input.Capability, input.IdempotencyKey, input.Body) + if err != nil { + return jobs.Job{}, false, err + } + command.Job.RetryOf = input.RetryOf + use := usageContext{Source: "platform", AccountID: session.User.ID, Username: session.User.Username, DisplayName: session.User.DisplayName, Role: session.User.Role, TenantID: session.User.TenantID, OrganizationID: session.User.OrganizationID, OrganizationName: session.User.OrganizationName} + command.Job.UsageContext, _ = json.Marshal(use) + parameters := map[string]any{} + var prepared providers.Request + if json.Unmarshal(command.Job.RequestPayload, &prepared) == nil { + for key, value := range prepared.Settings { + parameters[key] = value + } + parameters["inputUrlCount"] = len(prepared.InputURLs) + } + quote, err := c.quotes.Quote(ctx, billing.QuoteCommand{AccountID: session.User.ID, OrganizationID: session.User.OrganizationID, OrganizationName: session.User.OrganizationName, Role: session.User.Role, Provider: command.Job.Provider, Capability: command.Job.Capability, ReqKey: command.Job.ReqKey, Parameters: parameters, Payload: input.Body}) + if err != nil { + return jobs.Job{}, false, errors.New("quote generation charge") + } + if quote != nil { + state := quoteMap(*quote) + state["status"] = "pending" + command.Job.Billing, _ = json.Marshal(state) + command.HoldDispatch = true + } + created, reused, err := c.jobs.Create(ctx, command) + if err != nil { + return jobs.Job{}, false, err + } + return c.charge(ctx, created, reused) +} + +func (c *CreationCoordinator) CreatePublic(ctx context.Context, input CreationInput) (jobs.Job, bool, error) { + if c == nil || c.builder == nil || c.jobs == nil { + return jobs.Job{}, false, errors.New("create generation job") + } + command, err := c.builder.Build(ctx, input.OwnerID, input.ExternalClientID, input.Capability, input.IdempotencyKey, input.Body) + if err != nil { + return jobs.Job{}, false, err + } + return c.jobs.Create(ctx, command) +} + +// RetryPlatform mirrors the platform image retry contract: ownership and +// capability are checked by the HTTP adapter, while the new job is rebuilt +// through the normal builder/quote/charge pipeline with no inherited billing. +func (c *CreationCoordinator) RetryPlatform(ctx context.Context, session identity.Session, original jobs.Job) (jobs.Job, error) { + if original.OwnerID != session.User.ID || original.Capability != "image.generate" { + return jobs.Job{}, &jobs.Error{Kind: jobs.ErrorNotFound, Status: 404, Message: "任务不存在"} + } + var request providers.Request + if json.Unmarshal(original.RequestPayload, &request) != nil { + return jobs.Job{}, errors.New("retry generation job") + } + body := map[string]any{ + "prompt": request.Prompt, + "inputUrls": request.InputURLs, + "inputAssetIds": append([]string(nil), original.InputAssetIDs...), + "materials": request.Materials, + "settings": request.Settings, + "priority": jobs.NormalizePriority(original.Priority), + } + if original.WebhookURL != "" { + body["webhookUrl"] = original.WebhookURL + } + if engine := retryEngine(original.Provider, original.Capability); engine != "" { + body["engine"] = engine + } + created, _, err := c.CreatePlatform(ctx, session, CreationInput{Capability: original.Capability, Body: body, RetryOf: original.ID}) + if err != nil { + return jobs.Job{}, err + } + return created, nil +} + +func (c *CreationCoordinator) charge(ctx context.Context, job jobs.Job, reused bool) (jobs.Job, bool, error) { + if len(job.Billing) == 0 { + return job, reused, nil + } + var charge billingState + if json.Unmarshal(job.Billing, &charge) != nil { + return jobs.Job{}, reused, errors.New("charge generation job") + } + if charge.Status == "charged" || charge.Status == "not_charged" || charge.Status == "refunded" { + return job, reused, nil + } + if charge.Status != "pending" { + return jobs.Job{}, reused, errors.New("charge generation job") + } + if charge.QuotaExempt { + charge.raw["status"] = "not_charged" + return c.writeCharge(ctx, job, reused, charge.raw) + } + var use usageContext + if json.Unmarshal(job.UsageContext, &use) != nil || use.OrganizationID == "" || charge.AmountFen <= 0 { + return c.failCharge(ctx, job, reused, errors.New("charge generation job"), charge.raw) + } + request := billing.ChargeRequest{OrganizationID: use.OrganizationID, AccountID: use.AccountID, JobID: job.ID, AmountFen: charge.AmountFen, Description: capabilityLabel(job.Capability) + " · " + job.ReqKey, Metadata: map[string]any{"quote": cloneMap(charge.raw), "accountName": use.DisplayName, "organizationName": use.OrganizationName}} + if atomic, ok := c.state.(AtomicCreationCharger); ok { + pending, encodeErr := json.Marshal(charge.raw) + if encodeErr != nil { + return jobs.Job{}, reused, errors.New("charge generation job") + } + encoded, atomicErr := atomic.ChargeAndActivateCreation(ctx, request, pending) + if atomicErr != nil { + if errors.Is(atomicErr, billing.ErrCommitOutcomeUnknown) { + return jobs.Job{}, reused, errors.New("charge generation job") + } + return c.failCharge(ctx, job, reused, safeBillingError(atomicErr), charge.raw) + } + job.Billing = encoded + return job, reused, nil + } + posting, err := c.charges.Charge(ctx, request) + if err != nil { + return c.failCharge(ctx, job, reused, safeBillingError(err), charge.raw) + } + charge.raw["status"] = "charged" + charge.raw["ledgerEntryId"] = posting.LedgerID + charge.raw["chargedAt"] = posting.CreatedAt.UTC().Format("2006-01-02T15:04:05.999999999Z07:00") + return c.writeCharge(ctx, job, reused, charge.raw) +} + +func (c *CreationCoordinator) writeCharge(ctx context.Context, job jobs.Job, reused bool, state map[string]any) (jobs.Job, bool, error) { + encoded, err := json.Marshal(state) + if err != nil || c.state.ActivateCreation(ctx, job.ID, encoded) != nil { + return jobs.Job{}, reused, errors.New("persist generation charge") + } + job.Billing = encoded + return job, reused, nil +} + +func (c *CreationCoordinator) failCharge(ctx context.Context, job jobs.Job, reused bool, cause error, state map[string]any) (jobs.Job, bool, error) { + state["status"] = "not_charged" + job.Billing, _ = json.Marshal(state) + job.Status = jobs.StatusFailed + job.Error = &jobs.JobError{Message: "generation charge failed", Retryable: false} + if err := c.state.FailCreation(ctx, job); err != nil { + return jobs.Job{}, reused, errors.New("persist failed generation charge") + } + return job, reused, cause +} + +func quoteMap(quote billing.Quote) map[string]any { + raw, _ := json.Marshal(quote) + value := map[string]any{} + _ = json.Unmarshal(raw, &value) + return value +} + +func safeBillingError(err error) error { + switch billing.HTTPStatus(err) { + case 402: + return &billing.StatusError{Status: 402, Err: billing.ErrInsufficientBalance} + case 409: + return &billing.StatusError{Status: 409, Err: billing.ErrIdempotencyConflict} + default: + message := err.Error() + if strings.Contains(message, "BILLING_INSUFFICIENT_BALANCE") { + return &billing.StatusError{Status: 402, Err: billing.ErrInsufficientBalance} + } + if strings.Contains(message, "BILLING_IDEMPOTENCY_PAYLOAD_MISMATCH") { + return &billing.StatusError{Status: 409, Err: billing.ErrIdempotencyConflict} + } + return errors.New("charge generation job") + } +} + +func retryEngine(provider, capability string) string { + if capability == "video.generate" { + if provider == "seedance" || provider == "bailian" { + return provider + } + return "" + } + switch provider { + case "volcengine-visual": + return "jimeng" + case "evolink", "bailian": + return provider + default: + return "" + } +} diff --git a/backend/internal/orchestration/creation_test.go b/backend/internal/orchestration/creation_test.go new file mode 100644 index 0000000..419ef6c --- /dev/null +++ b/backend/internal/orchestration/creation_test.go @@ -0,0 +1,49 @@ +package orchestration + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" +) + +func TestCreationCoordinatorDoesNotFailCreationWhenAtomicCommitOutcomeIsUnknown(t *testing.T) { + state := &commitUnknownCreationState{} + coordinator := &CreationCoordinator{charges: &chargeLedgerStub{}, state: state} + job := jobs.Job{ + ID: "job-1", + Capability: "image.generate", + ReqKey: "wanx", + Billing: json.RawMessage(`{"status":"pending","amountFen":35}`), + UsageContext: json.RawMessage(`{"accountId":"account","organizationId":"org"}`), + } + + _, _, err := coordinator.charge(context.Background(), job, false) + if err == nil || err.Error() != "charge generation job" { + t.Fatalf("err=%v, want generic infrastructure error", err) + } + if state.failCalls != 0 { + t.Fatalf("FailCreation calls=%d, want 0", state.failCalls) + } +} + +type commitUnknownCreationState struct { + failCalls int +} + +func (*commitUnknownCreationState) ChargeAndActivateCreation(context.Context, billing.ChargeRequest, json.RawMessage) (json.RawMessage, error) { + return nil, errors.Join(errors.New("commit response and reconciliation failed"), billing.ErrCommitOutcomeUnknown) +} +func (*commitUnknownCreationState) WriteBilling(context.Context, string, json.RawMessage) error { + return nil +} +func (*commitUnknownCreationState) ActivateCreation(context.Context, string, json.RawMessage) error { + return nil +} +func (s *commitUnknownCreationState) FailCreation(context.Context, jobs.Job) error { + s.failCalls++ + return nil +} diff --git a/backend/internal/orchestration/orchestration.go b/backend/internal/orchestration/orchestration.go new file mode 100644 index 0000000..675bd60 --- /dev/null +++ b/backend/internal/orchestration/orchestration.go @@ -0,0 +1,377 @@ +// Package orchestration adapts jobs.Worker ports to the billing, usage, +// webhook, and asset modules. It contains cross-module policy but owns no +// persistence or external transport. +package orchestration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/webhook" +) + +type UsageSink interface { + Record(usage.Event) (*usage.Event, error) +} + +type UsageRecorder struct { + sink UsageSink + newID func() string + now func() time.Time +} + +func NewUsageRecorder(sink UsageSink, newID func() string, now func() time.Time) *UsageRecorder { + if now == nil { + now = time.Now + } + return &UsageRecorder{sink: sink, newID: newID, now: now} +} + +func (r *UsageRecorder) Record(_ context.Context, job jobs.Job) error { + if job.Provider == "mock" || job.ExternalClientID != "" { + return nil + } + var contextValue usageContext + if len(job.UsageContext) != 0 && json.Unmarshal(job.UsageContext, &contextValue) != nil { + return errors.New("record generation usage") + } + if contextValue.Source == "api" { + return nil + } + if r == nil || r.sink == nil || r.newID == nil { + return errors.New("record generation usage") + } + var charge billingState + if len(job.Billing) != 0 && json.Unmarshal(job.Billing, &charge) != nil { + return errors.New("record generation usage") + } + quantity := charge.Quantity + if quantity <= 0 { + quantity = 1 + } + unit := "job" + if charge.Unit == "image" || charge.Unit == "video_second" { + unit = charge.Unit + } + event := usage.Event{ + ID: r.newID(), OwnerID: job.OwnerID, JobID: job.ID, Source: "platform", + Capability: job.Capability, Provider: job.Provider, ReqKey: job.ReqKey, + AccountUsername: contextValue.Username, AccountDisplayName: contextValue.DisplayName, + TenantID: contextValue.TenantID, OrganizationID: contextValue.OrganizationID, + OrganizationName: contextValue.OrganizationName, Quantity: quantity, + EstimatedUnit: unit, ChargedAmountFen: charge.amountPointer(), Currency: charge.Currency, + CreatedAt: r.now().UTC().Format(time.RFC3339Nano), + } + if _, err := r.sink.Record(event); err != nil { + return errors.New("record generation usage") + } + return nil +} + +type RefundLedger interface { + Refund(context.Context, billing.RefundRequest) (billing.WalletPosting, error) +} + +// JobStateWriter is the deliberately narrow persistence seam needed because +// jobs.Patch does not expose billing or output_asset_ids fields. +type JobStateWriter interface { + WriteBilling(context.Context, string, json.RawMessage) error + WriteOutputAssetIDs(context.Context, string, []string) error +} + +type FencedJobStateWriter interface { + WriteBillingFenced(context.Context, string, json.RawMessage, jobs.Status, string) error + WriteOutputAssetIDsFenced(context.Context, string, []string, jobs.Status, string) error +} + +type TerminalRefund struct { + ledger RefundLedger + state JobStateWriter +} + +func NewTerminalRefund(ledger RefundLedger, state JobStateWriter) *TerminalRefund { + return &TerminalRefund{ledger: ledger, state: state} +} + +func (r *TerminalRefund) Refund(ctx context.Context, job jobs.Job, reason string) (jobs.Job, error) { + if !job.Status.Terminal() || job.Status == jobs.StatusSucceeded { + return job, nil + } + var charge billingState + if len(job.Billing) == 0 { + return job, nil + } + if err := json.Unmarshal(job.Billing, &charge); err != nil { + return jobs.Job{}, errors.New("refund generation charge") + } + if charge.Status != "charged" || charge.QuotaExempt { + return job, nil + } + var use usageContext + if json.Unmarshal(job.UsageContext, &use) != nil || use.OrganizationID == "" || charge.AmountFen <= 0 || r == nil || r.ledger == nil || r.state == nil { + return jobs.Job{}, errors.New("refund generation charge") + } + posting, err := r.ledger.Refund(ctx, billing.RefundRequest{ + OrganizationID: use.OrganizationID, AccountID: use.AccountID, JobID: job.ID, + AmountFen: charge.AmountFen, Description: capabilityLabel(job.Capability) + "失败退款 · " + reason, + Metadata: map[string]any{"chargeLedgerEntryId": charge.LedgerEntryID, "reason": reason, "quote": cloneMap(charge.raw)}, + }) + if err != nil { + return jobs.Job{}, errors.New("refund generation charge") + } + charge.raw["status"] = "refunded" + charge.raw["refundLedgerEntryId"] = posting.LedgerID + charge.raw["refundedAt"] = posting.CreatedAt.UTC().Format(time.RFC3339Nano) + charge.raw["refundReason"] = reason + encoded, err := json.Marshal(charge.raw) + if err != nil || writeBillingState(ctx, r.state, job, encoded) != nil { + return jobs.Job{}, errors.New("persist generation refund") + } + job.Billing = encoded + return job, nil +} + +type WebhookDeliverer interface { + Deliver(context.Context, jobs.Job) (webhook.Result, error) +} + +type WebhookBridge struct{ deliverer WebhookDeliverer } + +func NewWebhookBridge(deliverer WebhookDeliverer) *WebhookBridge { + return &WebhookBridge{deliverer: deliverer} +} + +func (b *WebhookBridge) Deliver(ctx context.Context, job jobs.Job) (jobs.WebhookResult, error) { + if b == nil || b.deliverer == nil { + return jobs.WebhookResult{}, errors.New("deliver generation webhook") + } + result, err := b.deliverer.Deliver(ctx, job) + if err != nil { + return jobs.WebhookResult{}, errors.New("deliver generation webhook") + } + return jobs.WebhookResult{Attempts: result.Attempts, LastStatus: result.LastStatus}, nil +} + +type OutputRegistrar interface { + Register(context.Context, jobs.Job) ([]string, error) +} + +type OutputRegisteringProcessor struct { + inner jobs.Processor + registrar OutputRegistrar + state JobStateWriter +} + +func NewOutputRegisteringProcessor(inner jobs.Processor, registrar OutputRegistrar, state JobStateWriter) *OutputRegisteringProcessor { + return &OutputRegisteringProcessor{inner: inner, registrar: registrar, state: state} +} + +func (p *OutputRegisteringProcessor) Advance(ctx context.Context, job jobs.Job) (jobs.Job, error) { + if job.Status == jobs.StatusSucceeded && len(job.OutputAssetIDs) != 0 { + return job, nil + } + if p == nil || p.inner == nil { + return jobs.Job{}, errors.New("advance generation job") + } + advanced, err := p.inner.Advance(ctx, job) + if err != nil || advanced.Status != jobs.StatusSucceeded || len(advanced.OutputAssetIDs) != 0 { + return advanced, err + } + if p.registrar == nil || p.state == nil { + return jobs.Job{}, errors.New("register generation outputs") + } + ids, err := p.registrar.Register(ctx, advanced) + if err != nil || len(ids) == 0 { + return jobs.Job{}, errors.New("register generation outputs") + } + if err := writeOutputState(ctx, p.state, advanced, ids); err != nil { + return jobs.Job{}, errors.New("persist generation outputs") + } + advanced.OutputAssetIDs = append([]string(nil), ids...) + return advanced, nil +} + +func writeBillingState(ctx context.Context, state JobStateWriter, job jobs.Job, value json.RawMessage) error { + if fenced, ok := state.(FencedJobStateWriter); ok && job.LockedBy != "" { + return fenced.WriteBillingFenced(ctx, job.ID, value, job.Status, job.LockedBy) + } + return state.WriteBilling(ctx, job.ID, value) +} + +func writeOutputState(ctx context.Context, state JobStateWriter, job jobs.Job, ids []string) error { + if fenced, ok := state.(FencedJobStateWriter); ok && job.LockedBy != "" { + return fenced.WriteOutputAssetIDsFenced(ctx, job.ID, ids, job.Status, job.LockedBy) + } + return state.WriteOutputAssetIDs(ctx, job.ID, ids) +} + +type GeneratedAssetImporter interface { + List(context.Context, assets.Scope) ([]assets.Asset, error) + ImportGenerated(context.Context, assets.Scope, assets.ImportGeneratedCommand) (assets.Asset, error) + ImportMock(context.Context, assets.Scope, assets.ImportMockCommand) (assets.Asset, error) +} + +type OutputURLResolver func(jobs.Job) ([]string, error) + +type AssetOutputRegistrar struct { + assets GeneratedAssetImporter + resolve OutputURLResolver +} + +func NewAssetOutputRegistrar(service GeneratedAssetImporter, resolve OutputURLResolver) *AssetOutputRegistrar { + return &AssetOutputRegistrar{assets: service, resolve: resolve} +} + +func (r *AssetOutputRegistrar) Register(ctx context.Context, job jobs.Job) ([]string, error) { + if r == nil || r.assets == nil || r.resolve == nil { + return nil, errors.New("register generation outputs") + } + scope := assets.PlatformScope(job.OwnerID) + existing, err := r.assets.List(ctx, scope) + if err != nil { + return nil, errors.New("register generation outputs") + } + if job.Provider == "mock" { + if id := existingOutputID(existing, job.ID, "output:0"); id != "" { + return []string{id}, nil + } + kind := assets.KindImage + if job.Capability == "video.generate" { + kind = assets.KindVideo + } + created, createErr := r.assets.ImportMock(ctx, scope, assets.ImportMockCommand{ + Capability: job.Capability, JobID: job.ID, Kind: kind, + Tags: []string{"generated", job.Capability, "job:" + job.ID, "output:0"}, + Metadata: map[string]any{"capability": job.Capability, "jobId": job.ID, "index": 0}, + }) + if createErr != nil { + return nil, errors.New("register generation outputs") + } + return []string{created.ID}, nil + } + urls, err := r.resolve(job) + if err != nil || len(urls) == 0 { + return nil, errors.New("register generation outputs") + } + ids := make([]string, 0, len(urls)) + for index, rawURL := range urls { + indexTag := fmt.Sprintf("output:%d", index) + if id := existingOutputID(existing, job.ID, indexTag); id != "" { + ids = append(ids, id) + continue + } + kind := assets.KindImage + if job.Capability == "video.generate" { + kind = assets.KindVideo + } + name := path.Base(strings.SplitN(rawURL, "?", 2)[0]) + if name == "." || name == "/" || name == "" { + name = fmt.Sprintf("%s-%d", strings.ReplaceAll(job.Capability, ".", "-"), index+1) + } + created, createErr := r.assets.ImportGenerated(ctx, scope, assets.ImportGeneratedCommand{ + URL: rawURL, Name: name, Kind: kind, Source: assets.SourceGenerated, + Tags: []string{"generated", job.Capability, "job:" + job.ID, indexTag}, + Metadata: map[string]any{"capability": job.Capability, "jobId": job.ID, "index": index}, + }) + if createErr != nil { + return nil, errors.New("register generation outputs") + } + ids = append(ids, created.ID) + } + return ids, nil +} + +func existingOutputID(existing []assets.Asset, jobID, indexTag string) string { + for _, asset := range existing { + if contains(asset.Tags, "job:"+jobID) && contains(asset.Tags, indexTag) { + return asset.ID + } + } + return "" +} + +func contains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +type usageContext struct { + Source string `json:"source"` + AccountID string `json:"accountId"` + Username string `json:"username"` + DisplayName string `json:"displayName"` + Role string `json:"role,omitempty"` + TenantID string `json:"tenantId"` + OrganizationID string `json:"organizationId"` + OrganizationName string `json:"organizationName"` +} + +type billingState struct { + Status, Unit, Currency, LedgerEntryID string + Quantity int + AmountFen int64 + QuotaExempt bool + raw map[string]any +} + +func (s *billingState) UnmarshalJSON(data []byte) error { + type alias billingState + var decoded struct { + Status, Unit, Currency, LedgerEntryID string + Quantity float64 + AmountFen int64 + QuotaExempt bool + } + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + if err := json.Unmarshal(data, &s.raw); err != nil { + return err + } + s.Status, s.Unit, s.Currency, s.LedgerEntryID = decoded.Status, decoded.Unit, decoded.Currency, decoded.LedgerEntryID + s.Quantity, s.AmountFen, s.QuotaExempt = int(decoded.Quantity), decoded.AmountFen, decoded.QuotaExempt + return nil +} + +func (s billingState) amountPointer() *int64 { + if _, exists := s.raw["amountFen"]; !exists { + return nil + } + value := s.AmountFen + return &value +} + +func capabilityLabel(capability string) string { + if capability == "video.generate" { + return "视频生成" + } + return "图片生成" +} + +func cloneMap(source map[string]any) map[string]any { + clone := make(map[string]any, len(source)) + for key, value := range source { + clone[key] = value + } + return clone +} + +var ( + _ jobs.UsageRecorder = (*UsageRecorder)(nil) + _ jobs.TerminalRefund = (*TerminalRefund)(nil) + _ jobs.WebhookDelivery = (*WebhookBridge)(nil) + _ jobs.Processor = (*OutputRegisteringProcessor)(nil) +) diff --git a/backend/internal/orchestration/orchestration_test.go b/backend/internal/orchestration/orchestration_test.go new file mode 100644 index 0000000..8c44682 --- /dev/null +++ b/backend/internal/orchestration/orchestration_test.go @@ -0,0 +1,519 @@ +package orchestration + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/providers" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/webhook" +) + +func TestUsageRecorderMapsPlatformJobAndSkipsPublicJobs(t *testing.T) { + sink := &usageSinkStub{} + recorder := NewUsageRecorder(sink, func() string { return "usage-1" }, func() time.Time { + return time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + }) + job := jobs.Job{ID: "job-1", OwnerID: "owner-1", Capability: "image.generate", Provider: "bailian", ReqKey: "wanx", UsageContext: json.RawMessage(`{"source":"platform","username":"alice","displayName":"Alice","tenantId":"tenant-1","organizationId":"org-1","organizationName":"Org"}`), Billing: json.RawMessage(`{"quantity":2,"unit":"image","amountFen":35,"currency":"CNY"}`)} + if err := recorder.Record(context.Background(), job); err != nil { + t.Fatal(err) + } + want := usage.Event{ID: "usage-1", OwnerID: "owner-1", JobID: "job-1", Source: "platform", Capability: "image.generate", Provider: "bailian", ReqKey: "wanx", AccountUsername: "alice", AccountDisplayName: "Alice", TenantID: "tenant-1", OrganizationID: "org-1", OrganizationName: "Org", Quantity: 2, EstimatedUnit: "image", ChargedAmountFen: int64Pointer(35), Currency: "CNY", CreatedAt: "2026-08-13T08:00:00Z"} + if !reflect.DeepEqual(sink.event, want) { + t.Fatalf("event = %#v, want %#v", sink.event, want) + } + if err := recorder.Record(context.Background(), jobs.Job{ID: "public", ExternalClientID: "partner"}); err != nil || sink.calls != 1 { + t.Fatalf("public Record err=%v calls=%d", err, sink.calls) + } +} + +func TestUsageRecorderReturnsGenericError(t *testing.T) { + recorder := NewUsageRecorder(&usageSinkStub{err: errors.New("postgres password leaked")}, func() string { return "id" }, time.Now) + err := recorder.Record(context.Background(), jobs.Job{ID: "job", Provider: "bailian"}) + if err == nil || err.Error() != "record generation usage" { + t.Fatalf("error = %v", err) + } +} + +func TestTerminalRefundUsesLedgerKeyAndPersistsRefundedBilling(t *testing.T) { + poster := &walletPosterStub{posting: billing.WalletPosting{LedgerID: "refund-ledger", CreatedAt: time.Date(2026, 8, 13, 8, 1, 0, 0, time.UTC)}} + ledger := billing.Ledger{Poster: poster, NewID: func() string { return "refund-ledger" }} + writer := &stateWriterStub{} + adapter := NewTerminalRefund(ledger, writer) + job := jobs.Job{ID: "job-1", OwnerID: "owner", Capability: "video.generate", Status: jobs.StatusFailed, UsageContext: json.RawMessage(`{"accountId":"account-1","organizationId":"org-1"}`), Billing: json.RawMessage(`{"status":"charged","amountFen":88,"ledgerEntryId":"charge-ledger","quotaExempt":false}`)} + got, err := adapter.Refund(context.Background(), job, "provider failed") + if err != nil { + t.Fatal(err) + } + if poster.params.JobID != "job-1" || poster.params.DeltaFen != 88 || poster.params.OrganizationID != "org-1" || poster.params.AccountID != "account-1" || poster.params.IdempotencyKey != "job-refund:job-1" { + t.Fatalf("refund posting = %#v", poster.params) + } + if poster.params.Description != "视频生成失败退款 · provider failed" || writer.jobID != "job-1" { + t.Fatalf("description=%q writer=%q", poster.params.Description, writer.jobID) + } + quote, _ := poster.params.Metadata["quote"].(map[string]any) + if quote["status"] != "charged" { + t.Fatalf("refund metadata quote mutated = %#v", quote) + } + var state map[string]any + if err := json.Unmarshal(got.Billing, &state); err != nil { + t.Fatal(err) + } + if state["status"] != "refunded" || state["refundLedgerEntryId"] != "refund-ledger" || state["refundReason"] != "provider failed" { + t.Fatalf("billing = %#v", state) + } +} + +func TestTerminalRefundNoopsUnlessChargedAndHidesDependencyErrors(t *testing.T) { + ledger := &refundLedgerStub{err: errors.New("wallet schema secret")} + writer := &stateWriterStub{} + adapter := NewTerminalRefund(ledger, writer) + unchanged := jobs.Job{ID: "pending", Status: jobs.StatusFailed, Billing: json.RawMessage(`{"status":"pending","amountFen":10}`)} + if got, err := adapter.Refund(context.Background(), unchanged, "x"); err != nil || string(got.Billing) != string(unchanged.Billing) || ledger.calls != 0 { + t.Fatalf("noop = %#v, %v calls=%d", got, err, ledger.calls) + } + charged := jobs.Job{ID: "charged", Status: jobs.StatusFailed, UsageContext: json.RawMessage(`{"organizationId":"org"}`), Billing: json.RawMessage(`{"status":"charged","amountFen":10}`)} + if _, err := adapter.Refund(context.Background(), charged, "x"); err == nil || err.Error() != "refund generation charge" { + t.Fatalf("error = %v", err) + } +} + +func TestWebhookBridgePreservesResult(t *testing.T) { + deliverer := &webhookStub{result: webhook.Result{Attempts: 2, LastStatus: &webhook.LastStatus{OK: true, Status: 204}}} + bridge := NewWebhookBridge(deliverer) + got, err := bridge.Deliver(context.Background(), jobs.Job{ID: "job"}) + if err != nil || got.Attempts != 2 || !reflect.DeepEqual(got.LastStatus, deliverer.result.LastStatus) { + t.Fatalf("Deliver = %#v, %v", got, err) + } +} + +func TestOutputRegisteringProcessorRegistersAndPersistsSuccessOutputs(t *testing.T) { + inner := &processorStub{job: jobs.Job{ID: "job-1", OwnerID: "owner", Capability: "image.generate", Status: jobs.StatusSucceeded}} + registrar := &outputRegistrarStub{ids: []string{"asset-1", "asset-2"}} + writer := &stateWriterStub{} + processor := NewOutputRegisteringProcessor(inner, registrar, writer) + got, err := processor.Advance(context.Background(), jobs.Job{ID: "job-1"}) + if err != nil || !reflect.DeepEqual(got.OutputAssetIDs, registrar.ids) || !reflect.DeepEqual(writer.outputIDs, registrar.ids) { + t.Fatalf("Advance = %#v, %v writer=%#v", got, err, writer.outputIDs) + } + if _, err := processor.Advance(context.Background(), got); err != nil || registrar.calls != 1 { + t.Fatalf("idempotent Advance error=%v calls=%d", err, registrar.calls) + } +} + +func TestAssetOutputRegistrarUsesGeneratedOwnerScopedAssets(t *testing.T) { + creator := &assetCreatorStub{} + registrar := NewAssetOutputRegistrar(creator, func(job jobs.Job) ([]string, error) { + return []string{"https://cdn.test/one.png", "https://cdn.test/two.png"}, nil + }) + ids, err := registrar.Register(context.Background(), jobs.Job{ID: "job-1", OwnerID: "owner", Capability: "image.generate"}) + if err != nil || !reflect.DeepEqual(ids, []string{"asset-1", "asset-2"}) { + t.Fatalf("Register = %#v, %v", ids, err) + } + if creator.scopes[0] != assets.PlatformScope("owner") || creator.commands[0].Source != assets.SourceGenerated || !reflect.DeepEqual(creator.commands[0].Tags, []string{"generated", "image.generate", "job:job-1", "output:0"}) { + t.Fatalf("asset request = %#v %#v", creator.scopes[0], creator.commands[0]) + } + creator.existing = []assets.Asset{{ID: "asset-existing", Tags: []string{"job:job-1", "output:0"}}} + ids, err = registrar.Register(context.Background(), jobs.Job{ID: "job-1", OwnerID: "owner", Capability: "image.generate"}) + if err != nil || ids[0] != "asset-existing" || len(creator.commands) != 3 { + t.Fatalf("retry Register = %#v, %v creates=%d", ids, err, len(creator.commands)) + } +} + +func TestOutputRegisteringProcessorCreatesAndPersistsMockAssetWithoutRemoteFetch(t *testing.T) { + for _, capability := range []string{"image.generate", "video.generate"} { + t.Run(capability, func(t *testing.T) { + creator := &assetCreatorStub{} + registrar := NewAssetOutputRegistrar(creator, ResolveProviderOutputURLs) + completed := jobs.Job{ + ID: "job-mock", OwnerID: "owner", Provider: "mock", Capability: capability, + Prompt: "mock output", Status: jobs.StatusSucceeded, + ResponsePayload: json.RawMessage(`{"status":"succeeded","outputUrls":["/generated-results/mock-task"]}`), + } + writer := &stateWriterStub{} + processor := NewOutputRegisteringProcessor(&processorStub{job: completed}, registrar, writer) + got, err := processor.Advance(context.Background(), jobs.Job{ID: "job-mock"}) + if err != nil || !reflect.DeepEqual(got.OutputAssetIDs, []string{"asset-1"}) || !reflect.DeepEqual(writer.outputIDs, []string{"asset-1"}) || len(creator.mockCommands) != 1 || creator.mockCommands[0].JobID != "job-mock" { + t.Fatalf("Advance = %#v, %v mock=%#v persisted=%#v", got, err, creator.mockCommands, writer.outputIDs) + } + wantKind := assets.KindImage + if capability == "video.generate" { + wantKind = assets.KindVideo + } + if creator.mockCommands[0].Kind != wantKind || len(creator.commands) != 0 { + t.Fatalf("mock command=%#v remote imports=%#v", creator.mockCommands[0], creator.commands) + } + }) + } +} + +func TestResolveProviderOutputURLsAcceptsCurrentShapesAndDeduplicates(t *testing.T) { + job := jobs.Job{ResponsePayload: json.RawMessage(`{"data":{"image_urls":["https://cdn.test/a.png","javascript:alert(1)"],"results":[{"url":"https://cdn.test/a.png"},{"url":"http://cdn.test/b.png"}]},"unrelated":"https://secret.test/not-output"}`)} + got, err := ResolveProviderOutputURLs(job) + if err != nil || !reflect.DeepEqual(got, []string{"https://cdn.test/a.png", "http://cdn.test/b.png"}) { + t.Fatalf("ResolveProviderOutputURLs = %#v, %v", got, err) + } + if _, err := ResolveProviderOutputURLs(jobs.Job{ResponsePayload: json.RawMessage(`{"status":"done"}`)}); err == nil || err.Error() != "resolve generation outputs" { + t.Fatalf("empty error = %v", err) + } + if _, err := ResolveProviderOutputURLs(jobs.Job{ResponsePayload: json.RawMessage(`{"outputUrls":["/generated-results/mock-task"]}`)}); err == nil || err.Error() != "resolve generation outputs" { + t.Fatalf("relative URL error = %v", err) + } +} + +func TestCreationCoordinatorQuotesPersistsThenChargesPlatformJob(t *testing.T) { + order := []string{} + creator := &creationStoreStub{order: &order} + quoter := "erStub{order: &order, quote: &billing.Quote{Provider: "bailian", Capability: "image.generate", ReqKey: "wanx", Unit: billing.UnitImage, Quantity: 2, AmountFen: 35, Currency: "CNY"}} + charger := &chargeLedgerStub{order: &order, posting: billing.WalletPosting{LedgerID: "charge-ledger", CreatedAt: time.Date(2026, 8, 13, 8, 2, 0, 0, time.UTC)}} + state := &creationStateStub{order: &order} + coordinator := NewCreationCoordinator(platformBuilderStub{}, creator, quoter, charger, state) + session := identity.Session{User: identity.User{ID: "account-1", Username: "alice", DisplayName: "Alice", TenantID: "tenant-1", OrganizationID: "org-1", OrganizationName: "Org", Role: "user"}} + created, reused, err := coordinator.CreatePlatform(context.Background(), session, CreationInput{Capability: "image.generate", Body: map[string]any{"prompt": "hello", "settings": map[string]any{"imageCount": 2}}, IdempotencyKey: "idem"}) + if err != nil || reused || !reflect.DeepEqual(order, []string{"quote", "create", "charge", "billing"}) { + t.Fatalf("CreatePlatform = %#v,%v,%v order=%#v", created, reused, err, order) + } + if charger.request.JobID != "job-1" || charger.request.AmountFen != 35 || charger.request.OrganizationID != "org-1" || charger.request.AccountID != "account-1" { + t.Fatalf("charge = %#v", charger.request) + } + var use map[string]any + _ = json.Unmarshal(created.UsageContext, &use) + if use["username"] != "alice" || use["organizationId"] != "org-1" || use["source"] != "platform" { + t.Fatalf("usageContext = %#v", use) + } + var charged map[string]any + _ = json.Unmarshal(created.Billing, &charged) + if charged["status"] != "charged" || charged["ledgerEntryId"] != "charge-ledger" { + t.Fatalf("billing = %#v", charged) + } + if quoter.command.Parameters["imageCount"] != float64(2) { + t.Fatalf("quote parameters = %#v", quoter.command.Parameters) + } +} + +func TestCreationCoordinatorMarksPersistedJobFailedWhenChargeFails(t *testing.T) { + creator := &creationStoreStub{} + quoter := "erStub{quote: &billing.Quote{AmountFen: 35, Currency: "CNY"}} + charger := &chargeLedgerStub{err: &billing.StatusError{Status: 402, Err: billing.ErrInsufficientBalance}} + state := &creationStateStub{} + coordinator := NewCreationCoordinator(platformBuilderStub{}, creator, quoter, charger, state) + session := identity.Session{User: identity.User{ID: "account", DisplayName: "A", OrganizationID: "org", Role: "user"}} + _, _, err := coordinator.CreatePlatform(context.Background(), session, CreationInput{Capability: "image.generate", Body: map[string]any{}}) + if billing.HTTPStatus(err) != 402 || state.failed.ID != "job-1" || state.failed.Status != jobs.StatusFailed { + t.Fatalf("error=%v failed=%#v", err, state.failed) + } + var failedBilling map[string]any + _ = json.Unmarshal(state.failed.Billing, &failedBilling) + if failedBilling["status"] != "not_charged" { + t.Fatalf("failed billing = %#v", failedBilling) + } +} + +func TestCreationCoordinatorReplaysPendingChargeWithSameJobID(t *testing.T) { + pending := jobs.Job{ID: "existing-job", OwnerID: "account", Provider: "bailian", Capability: "image.generate", ReqKey: "wanx", Status: jobs.StatusQueued, UsageContext: json.RawMessage(`{"accountId":"account","organizationId":"org"}`), Billing: json.RawMessage(`{"status":"pending","amountFen":35,"currency":"CNY"}`)} + creator := &creationStoreStub{existing: &pending} + quoter := "erStub{quote: &billing.Quote{AmountFen: 35, Currency: "CNY"}} + charger := &chargeLedgerStub{posting: billing.WalletPosting{LedgerID: "same-key-ledger"}} + coordinator := NewCreationCoordinator(platformBuilderStub{}, creator, quoter, charger, &creationStateStub{}) + created, reused, err := coordinator.CreatePlatform(context.Background(), identity.Session{User: identity.User{ID: "account", DisplayName: "A", OrganizationID: "org"}}, CreationInput{Capability: "image.generate", Body: map[string]any{}, IdempotencyKey: "same"}) + if err != nil || !reused || created.ID != "existing-job" || charger.request.JobID != "existing-job" { + t.Fatalf("replay = %#v,%v,%v charge=%#v", created, reused, err, charger.request) + } +} + +func TestCreationCoordinatorKeepsPendingChargeUndispatchableAndActivatesAfterDurableCharge(t *testing.T) { + creator := &creationStoreStub{} + quoter := "erStub{quote: &billing.Quote{AmountFen: 35, Currency: "CNY"}} + charger := &chargeLedgerStub{posting: billing.WalletPosting{LedgerID: "charge-ledger"}} + state := &creationStateStub{} + coordinator := NewCreationCoordinator(platformBuilderStub{}, creator, quoter, charger, state) + + created, _, err := coordinator.CreatePlatform(context.Background(), identity.Session{User: identity.User{ID: "account", OrganizationID: "org"}}, CreationInput{Capability: "image.generate", Body: map[string]any{"prompt": "fresh"}}) + if err != nil { + t.Fatal(err) + } + if creator.created.DispatchReadyAt != nil { + t.Fatalf("pending creation was dispatchable: %#v", creator.created) + } + if state.activatedID != created.ID { + t.Fatalf("activation = %q, want %q", state.activatedID, created.ID) + } +} + +func TestCreationCoordinatorUsesAtomicChargeAndActivationWhenStoreSupportsIt(t *testing.T) { + creator := &creationStoreStub{} + quoter := "erStub{quote: &billing.Quote{AmountFen: 35, Currency: "CNY"}} + state := &atomicCreationStateStub{charged: json.RawMessage(`{"status":"charged","amountFen":35,"ledgerEntryId":"atomic"}`)} + charger := &chargeLedgerStub{} + coordinator := NewCreationCoordinator(platformBuilderStub{}, creator, quoter, charger, state) + created, _, err := coordinator.CreatePlatform(context.Background(), identity.Session{User: identity.User{ID: "account", OrganizationID: "org"}}, CreationInput{Capability: "image.generate", Body: map[string]any{"prompt": "fresh"}}) + if err != nil || state.calls != 1 || charger.calls != 0 || string(created.Billing) != string(state.charged) { + t.Fatalf("created=%#v err=%v atomic=%d fallback=%d", created, err, state.calls, charger.calls) + } +} + +func TestCreationCoordinatorRetryRebuildsAnyOwnedImageJobWithFreshQuote(t *testing.T) { + creator := &creationStoreStub{} + quoter := "erStub{quote: &billing.Quote{AmountFen: 41, Currency: "CNY"}} + charger := &chargeLedgerStub{posting: billing.WalletPosting{LedgerID: "fresh-ledger"}} + state := &creationStateStub{} + coordinator := NewCreationCoordinator(platformBuilderStub{}, creator, quoter, charger, state) + original := jobs.Job{ID: "old", OwnerID: "account", Capability: "image.generate", Status: jobs.StatusRunning, Priority: 73, WebhookURL: "https://hooks.test/result", InputAssetIDs: []string{"asset-1"}, Billing: json.RawMessage(`{"status":"charged","amountFen":999}`), RequestPayload: json.RawMessage(`{"capability":"image.generate","model":"wanx","prompt":"again","inputUrls":["https://in.test/a.png"],"settings":{"imageCount":2}}`)} + + retried, err := coordinator.RetryPlatform(context.Background(), identity.Session{User: identity.User{ID: "account", OrganizationID: "org"}}, original) + if err != nil { + t.Fatal(err) + } + if retried.RetryOf != "old" || retried.ID == "old" || charger.request.AmountFen != 41 { + t.Fatalf("retry=%#v charge=%#v", retried, charger.request) + } + var fresh map[string]any + _ = json.Unmarshal(retried.Billing, &fresh) + if fresh["amountFen"] != float64(41) || fresh["ledgerEntryId"] != "fresh-ledger" { + t.Fatalf("fresh billing = %#v", fresh) + } + if !reflect.DeepEqual(quoter.command.Payload["inputAssetIds"], []string{"asset-1"}) || quoter.command.Payload["webhookUrl"] != "https://hooks.test/result" || quoter.command.Payload["priority"] != 73 { + t.Fatalf("retry payload = %#v", quoter.command.Payload) + } +} + +func TestSafeBillingErrorMapsPostgresWalletFailures(t *testing.T) { + tests := []struct { + name string + err error + want int + }{ + {name: "insufficient", err: errors.New("charge and activate: BILLING_INSUFFICIENT_BALANCE"), want: 402}, + {name: "idempotency", err: errors.New("charge and activate: BILLING_IDEMPOTENCY_PAYLOAD_MISMATCH"), want: 409}, + {name: "unknown", err: errors.New("database unavailable"), want: 500}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := safeBillingError(test.err) + if status := billing.HTTPStatus(got); status != test.want { + t.Fatalf("status = %d, want %d (error %v)", status, test.want, got) + } + if test.want == 500 && got.Error() != "charge generation job" { + t.Fatalf("unknown error leaked: %v", got) + } + }) + } +} + +func TestCreationCoordinatorPublicCreationNeverQuotesOrCharges(t *testing.T) { + creator := &creationStoreStub{} + quoter := "erStub{} + charger := &chargeLedgerStub{} + coordinator := NewCreationCoordinator(platformBuilderStub{}, creator, quoter, charger, &creationStateStub{}) + created, _, err := coordinator.CreatePublic(context.Background(), CreationInput{OwnerID: "api:partner", ExternalClientID: "partner", Capability: "image.generate", Body: map[string]any{}}) + if err != nil || created.ExternalClientID != "partner" || quoter.calls != 0 || charger.calls != 0 { + t.Fatalf("CreatePublic = %#v,%v quote=%d charge=%d", created, err, quoter.calls, charger.calls) + } +} + +type usageSinkStub struct { + event usage.Event + calls int + err error +} + +func (s *usageSinkStub) Record(event usage.Event) (*usage.Event, error) { + s.calls++ + s.event = event + return &event, s.err +} + +type refundLedgerStub struct { + request billing.RefundRequest + posting billing.WalletPosting + calls int + err error +} + +type walletPosterStub struct { + params billing.WalletPostParams + posting billing.WalletPosting + err error +} + +func (s *walletPosterStub) PostWalletEntry(_ context.Context, params billing.WalletPostParams) (billing.WalletPosting, error) { + s.params = params + return s.posting, s.err +} + +func (s *refundLedgerStub) Refund(_ context.Context, request billing.RefundRequest) (billing.WalletPosting, error) { + s.calls++ + s.request = request + return s.posting, s.err +} + +type stateWriterStub struct { + jobID string + billing json.RawMessage + outputIDs []string + err error +} + +func (s *stateWriterStub) WriteBilling(_ context.Context, id string, value json.RawMessage) error { + s.jobID = id + s.billing = append([]byte(nil), value...) + return s.err +} +func (s *stateWriterStub) WriteOutputAssetIDs(_ context.Context, id string, ids []string) error { + s.jobID = id + s.outputIDs = append([]string(nil), ids...) + return s.err +} + +type webhookStub struct { + result webhook.Result + err error +} + +func (s *webhookStub) Deliver(context.Context, jobs.Job) (webhook.Result, error) { + return s.result, s.err +} + +type processorStub struct { + job jobs.Job + err error +} + +func (s *processorStub) Advance(context.Context, jobs.Job) (jobs.Job, error) { return s.job, s.err } + +type outputRegistrarStub struct { + ids []string + calls int + err error +} + +func (s *outputRegistrarStub) Register(context.Context, jobs.Job) ([]string, error) { + s.calls++ + return append([]string(nil), s.ids...), s.err +} + +type assetCreatorStub struct { + scopes []assets.Scope + commands []assets.ImportGeneratedCommand + mockCommands []assets.ImportMockCommand + existing []assets.Asset +} + +func (s *assetCreatorStub) List(_ context.Context, _ assets.Scope) ([]assets.Asset, error) { + return append([]assets.Asset(nil), s.existing...), nil +} + +func (s *assetCreatorStub) ImportGenerated(_ context.Context, scope assets.Scope, command assets.ImportGeneratedCommand) (assets.Asset, error) { + s.scopes = append(s.scopes, scope) + s.commands = append(s.commands, command) + return assets.Asset{ID: "asset-" + string(rune('0'+len(s.commands)))}, nil +} + +func (s *assetCreatorStub) ImportMock(_ context.Context, scope assets.Scope, command assets.ImportMockCommand) (assets.Asset, error) { + s.scopes = append(s.scopes, scope) + s.mockCommands = append(s.mockCommands, command) + return assets.Asset{ID: "asset-" + string(rune('0'+len(s.mockCommands)))}, nil +} + +type platformBuilderStub struct{} + +func (platformBuilderStub) Build(_ context.Context, owner, client, capability, idempotency string, body map[string]any) (jobs.CreateCommand, error) { + settings, _ := body["settings"].(map[string]any) + request, _ := json.Marshal(providers.Request{Capability: capability, Model: "wanx", Settings: settings}) + return jobs.CreateCommand{Job: jobs.Job{ID: "job-1", OwnerID: owner, ExternalClientID: client, Capability: capability, Provider: "bailian", ReqKey: "wanx", Status: jobs.StatusQueued, IdempotencyKey: idempotency, RequestPayload: request}, IdempotencyBody: body}, nil +} + +type creationStoreStub struct { + order *[]string + existing *jobs.Job + created jobs.Job +} + +func (s *creationStoreStub) Create(_ context.Context, command jobs.CreateCommand) (jobs.Job, bool, error) { + if s.order != nil { + *s.order = append(*s.order, "create") + } + if s.existing != nil { + return *s.existing, true, nil + } + s.created = command.Job + return command.Job, false, nil +} + +type quoterStub struct { + order *[]string + quote *billing.Quote + err error + calls int + command billing.QuoteCommand +} + +func (s *quoterStub) Quote(_ context.Context, command billing.QuoteCommand) (*billing.Quote, error) { + s.calls++ + s.command = command + if s.order != nil { + *s.order = append(*s.order, "quote") + } + return s.quote, s.err +} + +type chargeLedgerStub struct { + order *[]string + request billing.ChargeRequest + posting billing.WalletPosting + err error + calls int +} + +func (s *chargeLedgerStub) Charge(_ context.Context, request billing.ChargeRequest) (billing.WalletPosting, error) { + s.calls++ + s.request = request + if s.order != nil { + *s.order = append(*s.order, "charge") + } + return s.posting, s.err +} + +type creationStateStub struct { + order *[]string + failed jobs.Job + activatedID string +} + +type atomicCreationStateStub struct { + creationStateStub + charged json.RawMessage + calls int +} + +func (s *atomicCreationStateStub) ChargeAndActivateCreation(_ context.Context, _ billing.ChargeRequest, _ json.RawMessage) (json.RawMessage, error) { + s.calls++ + return append(json.RawMessage(nil), s.charged...), nil +} + +func (s *creationStateStub) WriteBilling(_ context.Context, _ string, _ json.RawMessage) error { + if s.order != nil { + *s.order = append(*s.order, "billing") + } + return nil +} +func (s *creationStateStub) FailCreation(_ context.Context, job jobs.Job) error { + s.failed = job + return nil +} +func (s *creationStateStub) ActivateCreation(_ context.Context, id string, _ json.RawMessage) error { + s.activatedID = id + if s.order != nil { + *s.order = append(*s.order, "billing") + } + return nil +} +func int64Pointer(value int64) *int64 { return &value } diff --git a/backend/internal/orchestration/output_urls.go b/backend/internal/orchestration/output_urls.go new file mode 100644 index 0000000..be97da9 --- /dev/null +++ b/backend/internal/orchestration/output_urls.go @@ -0,0 +1,73 @@ +package orchestration + +import ( + "encoding/json" + "errors" + "net/url" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/providers" +) + +// ResolveProviderOutputURLs extracts only documented provider output fields; +// it does not recursively accept arbitrary strings from response payloads. +func ResolveProviderOutputURLs(job jobs.Job) ([]string, error) { + var root map[string]any + if len(job.ResponsePayload) == 0 || json.Unmarshal(job.ResponsePayload, &root) != nil { + return nil, errors.New("resolve generation outputs") + } + var persisted providers.HTTPResult + if json.Unmarshal(job.ResponsePayload, &persisted) == nil && len(persisted.OutputURLs) > 0 { + seen := map[string]bool{} + result := []string{} + for _, outputURL := range persisted.OutputURLs { + collectOutputURLs(outputURL, seen, &result) + } + if len(result) > 0 { + return result, nil + } + } + candidates := []any{} + data := objectMap(root["data"]) + output := objectMap(root["output"]) + content := objectMap(root["content"]) + dataContent := objectMap(data["content"]) + for _, object := range []map[string]any{root, data, output, content, dataContent} { + for _, key := range []string{"image_urls", "images", "results", "choices", "outputs", "video_url", "url"} { + candidates = append(candidates, object[key]) + } + } + seen := map[string]bool{} + result := []string{} + for _, candidate := range candidates { + collectOutputURLs(candidate, seen, &result) + } + if len(result) == 0 { + return nil, errors.New("resolve generation outputs") + } + return result, nil +} + +func collectOutputURLs(value any, seen map[string]bool, result *[]string) { + switch typed := value.(type) { + case string: + parsed, err := url.Parse(typed) + if err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && !seen[typed] { + seen[typed] = true + *result = append(*result, typed) + } + case []any: + for _, item := range typed { + collectOutputURLs(item, seen, result) + } + case map[string]any: + for _, key := range []string{"url", "image_url", "video_url"} { + collectOutputURLs(typed[key], seen, result) + } + } +} + +func objectMap(value any) map[string]any { + object, _ := value.(map[string]any) + return object +} diff --git a/backend/internal/orchestration/settlement.go b/backend/internal/orchestration/settlement.go new file mode 100644 index 0000000..6f5ed2d --- /dev/null +++ b/backend/internal/orchestration/settlement.go @@ -0,0 +1,220 @@ +package orchestration + +import ( + "context" + "encoding/json" + "errors" + "strings" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" +) + +type SettlementLedger interface { + Settle(context.Context, billing.SettlementRequest) (billing.WalletPosting, error) +} + +type SettlementStateWriter interface { + WriteBilling(context.Context, string, json.RawMessage) error +} + +type FencedSettlementStateWriter interface { + WriteBillingFenced(context.Context, string, json.RawMessage, jobs.Status, string) error +} + +// SettlementProcessor adds Seedance actual-usage settlement to a jobs +// processor. Other providers and non-successful advances pass through. +type SettlementProcessor struct { + inner jobs.Processor + ledger SettlementLedger + state SettlementStateWriter + now func() time.Time +} + +func NewSettlementProcessor(inner jobs.Processor, ledger SettlementLedger, state SettlementStateWriter, now func() time.Time) *SettlementProcessor { + if now == nil { + now = time.Now + } + return &SettlementProcessor{inner: inner, ledger: ledger, state: state, now: now} +} + +func (p *SettlementProcessor) Advance(ctx context.Context, job jobs.Job) (jobs.Job, error) { + if p == nil || p.inner == nil { + return jobs.Job{}, errors.New("advance generation job") + } + advanced, err := p.inner.Advance(ctx, job) + if err != nil || advanced.Provider != "seedance" || advanced.Status != jobs.StatusSucceeded || len(advanced.Billing) == 0 { + return advanced, err + } + + var charge billingState + if json.Unmarshal(advanced.Billing, &charge) != nil { + return jobs.Job{}, errors.New("settle seedance generation charge") + } + status, _ := charge.raw["settlementStatus"].(string) + if status == "settled" || status == "estimated" { + return advanced, nil + } + var use usageContext + if len(advanced.UsageContext) != 0 && json.Unmarshal(advanced.UsageContext, &use) != nil { + return jobs.Job{}, errors.New("settle seedance generation charge") + } + quotaExempt := charge.QuotaExempt || use.Source == "platform" && use.Role == "super_admin" + chargeReady := charge.Status == "charged" || quotaExempt && charge.Status == "not_charged" + if !chargeReady || !quotaExempt && use.OrganizationID == "" { + return advanced, nil + } + + completionTokens := seedanceCompletionTokens(advanced.ResponsePayload) + if completionTokens <= 0 { + charge.raw["settlementStatus"] = "estimated" + charge.raw["settlementReason"] = "provider_usage_unavailable" + charge.raw["settledAt"] = p.now().UTC().Format(time.RFC3339Nano) + return p.write(ctx, advanced, charge.raw) + } + + resolution := seedanceResolution(charge.raw) + inputVideo := seedanceRequestHasInputVideo(advanced.RequestPayload) + actualAmount, err := billing.CalculateSeedanceActualAmountFen(billing.SeedanceActualAmountInput{ + Resolution: resolution, InputVideo: inputVideo, CompletionTokens: completionTokens, + MarkupMultiplier: numberOrZero(charge.raw["markupMultiplier"]), + }) + if err != nil { + return jobs.Job{}, errors.New("settle seedance generation charge") + } + delta := actualAmount - charge.AmountFen + settledAt := p.now().UTC() + var settlementLedgerID string + if delta != 0 && !quotaExempt { + if p.ledger == nil { + return jobs.Job{}, errors.New("settle seedance generation charge") + } + description := capabilityLabel(advanced.Capability) + "实际用量差额退回" + if delta > 0 { + description = capabilityLabel(advanced.Capability) + "实际用量补扣" + } + reservedAmount := charge.AmountFen + if value, ok := integer(charge.raw["reservedAmountFen"]); ok { + reservedAmount = value + } + posting, postErr := p.ledger.Settle(ctx, billing.SettlementRequest{ + OrganizationID: use.OrganizationID, AccountID: use.AccountID, JobID: advanced.ID, + DeltaFen: delta, Description: description, + Metadata: map[string]any{ + "operation": "seedance_actual_settlement", "reservedAmountFen": reservedAmount, + "chargedAmountFen": charge.AmountFen, "actualAmountFen": actualAmount, + "completionTokens": completionTokens, "inputVideo": inputVideo, "resolution": resolution, + }, + }) + if postErr != nil { + return jobs.Job{}, errors.New("settle seedance generation charge") + } + settlementLedgerID = posting.LedgerID + if !posting.CreatedAt.IsZero() { + settledAt = posting.CreatedAt.UTC() + } + } + + charge.raw["amountFen"] = actualAmount + charge.raw["settlementStatus"] = "settled" + charge.raw["settledAt"] = settledAt.Format(time.RFC3339Nano) + if settlementLedgerID != "" { + charge.raw["settlementLedgerEntryId"] = settlementLedgerID + } else { + delete(charge.raw, "settlementLedgerEntryId") + } + charge.raw["providerUsage"] = map[string]any{ + "completionTokens": completionTokens, "resolution": resolution, "inputVideo": inputVideo, + "tokenPriceFenPerMillion": billing.SeedanceTokenPriceFenPerMillion(resolution, inputVideo), + } + return p.write(ctx, advanced, charge.raw) +} + +func (p *SettlementProcessor) write(ctx context.Context, job jobs.Job, snapshot map[string]any) (jobs.Job, error) { + if p.state == nil { + return jobs.Job{}, errors.New("persist seedance generation settlement") + } + encoded, err := json.Marshal(snapshot) + if err != nil { + return jobs.Job{}, errors.New("persist seedance generation settlement") + } + var writeErr error + if fenced, ok := p.state.(FencedSettlementStateWriter); ok && job.LockedBy != "" { + writeErr = fenced.WriteBillingFenced(ctx, job.ID, encoded, job.Status, job.LockedBy) + } else { + writeErr = p.state.WriteBilling(ctx, job.ID, encoded) + } + if writeErr != nil { + return jobs.Job{}, errors.New("persist seedance generation settlement") + } + job.Billing = encoded + return job, nil +} + +func seedanceCompletionTokens(payload json.RawMessage) int64 { + var response struct { + Usage map[string]any `json:"usage"` + } + if json.Unmarshal(payload, &response) != nil { + return 0 + } + value, _ := integer(response.Usage["completionTokens"]) + return value +} + +func seedanceResolution(snapshot map[string]any) string { + parameters, _ := snapshot["parameters"].(map[string]any) + resolution, _ := parameters["resolution"].(string) + resolution = strings.ToLower(strings.TrimSpace(resolution)) + if resolution == "480p" || resolution == "1080p" || resolution == "4k" { + return resolution + } + return "720p" +} + +func seedanceRequestHasInputVideo(payload json.RawMessage) bool { + var value any + if json.Unmarshal(payload, &value) != nil { + return false + } + return hasVideoMaterial(value) +} + +func hasVideoMaterial(value any) bool { + switch typed := value.(type) { + case map[string]any: + if materialType, _ := typed["type"].(string); strings.EqualFold(strings.TrimSpace(materialType), "video") { + return true + } + for _, child := range typed { + if hasVideoMaterial(child) { + return true + } + } + case []any: + for _, child := range typed { + if hasVideoMaterial(child) { + return true + } + } + } + return false +} + +func numberOrZero(value any) float64 { + if number, ok := value.(float64); ok { + return number + } + return 0 +} + +func integer(value any) (int64, bool) { + number, ok := value.(float64) + if !ok || number <= 0 { + return 0, false + } + return int64(number), true +} + +var _ jobs.Processor = (*SettlementProcessor)(nil) diff --git a/backend/internal/orchestration/settlement_test.go b/backend/internal/orchestration/settlement_test.go new file mode 100644 index 0000000..4b4992f --- /dev/null +++ b/backend/internal/orchestration/settlement_test.go @@ -0,0 +1,129 @@ +package orchestration + +import ( + "context" + "encoding/json" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs" +) + +func TestSettlementProcessorSettlesSuccessfulSeedanceActualUsage(t *testing.T) { + state := &settlementStateStub{} + ledger := &settlementLedgerStub{posting: billing.WalletPosting{LedgerID: "settlement-1", CreatedAt: time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)}} + advanced := seedanceSettlementJob(`{"usage":{"completionTokens":20000}}`, `{"status":"charged","amountFen":100,"reservedAmountFen":120,"markupMultiplier":1.2,"parameters":{"resolution":"720p"}}`) + processor := NewSettlementProcessor(settlementProcessorStub{job: advanced}, ledger, state, func() time.Time { return time.Date(2026, 8, 13, 9, 0, 0, 0, time.UTC) }) + + got, err := processor.Advance(context.Background(), jobs.Job{ID: "job-1", Provider: "seedance", Status: jobs.StatusRunning}) + if err != nil { + t.Fatal(err) + } + if ledger.calls != 1 || ledger.request.DeltaFen != 11 || ledger.request.JobID != "job-1" || ledger.request.OrganizationID != "org-1" { + t.Fatalf("settlement request = %#v, calls=%d", ledger.request, ledger.calls) + } + if ledger.request.Metadata["completionTokens"] != int64(20000) || ledger.request.Metadata["actualAmountFen"] != int64(111) { + t.Fatalf("settlement metadata = %#v", ledger.request.Metadata) + } + var snapshot map[string]any + if err := json.Unmarshal(got.Billing, &snapshot); err != nil { + t.Fatal(err) + } + usage := snapshot["providerUsage"].(map[string]any) + if snapshot["settlementStatus"] != "settled" || snapshot["amountFen"] != float64(111) || snapshot["settlementLedgerEntryId"] != "settlement-1" || usage["completionTokens"] != float64(20000) || state.calls != 1 { + t.Fatalf("billing snapshot = %#v, writes=%d", snapshot, state.calls) + } +} + +func TestSettlementProcessorMarksMissingProviderUsageEstimated(t *testing.T) { + state := &settlementStateStub{} + advanced := seedanceSettlementJob(`{"usage":{}}`, `{"status":"charged","amountFen":100,"markupMultiplier":1.2,"parameters":{"resolution":"720p"}}`) + processor := NewSettlementProcessor(settlementProcessorStub{job: advanced}, &settlementLedgerStub{}, state, func() time.Time { return time.Date(2026, 8, 13, 9, 0, 0, 0, time.UTC) }) + + got, err := processor.Advance(context.Background(), jobs.Job{ID: "job-1", Provider: "seedance", Status: jobs.StatusRunning}) + if err != nil { + t.Fatal(err) + } + var snapshot map[string]any + _ = json.Unmarshal(got.Billing, &snapshot) + if snapshot["settlementStatus"] != "estimated" || snapshot["settlementReason"] != "provider_usage_unavailable" || state.calls != 1 { + t.Fatalf("billing snapshot = %#v, writes=%d", snapshot, state.calls) + } +} + +func TestSettlementProcessorQuotaExemptUpdatesSnapshotWithoutWalletAndIsIdempotent(t *testing.T) { + state := &settlementStateStub{} + ledger := &settlementLedgerStub{} + advanced := seedanceSettlementJob(`{"usage":{"completionTokens":20000}}`, `{"status":"not_charged","amountFen":100,"markupMultiplier":1.2,"quotaExempt":true,"parameters":{"resolution":"720p"}}`) + processor := NewSettlementProcessor(settlementProcessorStub{job: advanced}, ledger, state, time.Now) + + got, err := processor.Advance(context.Background(), jobs.Job{ID: "job-1", Provider: "seedance", Status: jobs.StatusRunning}) + if err != nil || ledger.calls != 0 || state.calls != 1 { + t.Fatalf("first settlement calls ledger=%d state=%d err=%v", ledger.calls, state.calls, err) + } + processor.inner = settlementProcessorStub{job: got} + got, err = processor.Advance(context.Background(), got) + if err != nil || ledger.calls != 0 || state.calls != 1 { + t.Fatalf("repeat settlement calls ledger=%d state=%d err=%v", ledger.calls, state.calls, err) + } + var snapshot map[string]any + _ = json.Unmarshal(got.Billing, &snapshot) + if snapshot["amountFen"] != float64(111) || snapshot["settlementStatus"] != "settled" { + t.Fatalf("billing snapshot = %#v", snapshot) + } +} + +func TestSettlementProcessorLeavesNonSeedanceSuccessAlone(t *testing.T) { + state := &settlementStateStub{} + advanced := jobs.Job{ID: "job-2", Provider: "bailian", Status: jobs.StatusSucceeded, Billing: json.RawMessage(`{"status":"charged"}`)} + got, err := NewSettlementProcessor(settlementProcessorStub{job: advanced}, &settlementLedgerStub{}, state, time.Now).Advance(context.Background(), jobs.Job{ID: "job-2"}) + if err != nil || got.Provider != "bailian" || state.calls != 0 { + t.Fatalf("job=%#v writes=%d err=%v", got, state.calls, err) + } +} + +func seedanceSettlementJob(response, billingJSON string) jobs.Job { + return jobs.Job{ + ID: "job-1", Provider: "seedance", Capability: "video.generate", Status: jobs.StatusSucceeded, + RequestPayload: json.RawMessage(`{"settings":{"resolution":"720p"},"inputUrls":[]}`), + ResponsePayload: json.RawMessage(response), Billing: json.RawMessage(billingJSON), + UsageContext: json.RawMessage(`{"organizationId":"org-1","accountId":"account-1"}`), + } +} + +type settlementProcessorStub struct { + job jobs.Job + err error +} + +func (s settlementProcessorStub) Advance(context.Context, jobs.Job) (jobs.Job, error) { + return s.job, s.err +} + +type settlementLedgerStub struct { + request billing.SettlementRequest + posting billing.WalletPosting + err error + calls int +} + +func (s *settlementLedgerStub) Settle(_ context.Context, request billing.SettlementRequest) (billing.WalletPosting, error) { + s.calls++ + s.request = request + return s.posting, s.err +} + +type settlementStateStub struct { + id string + billing json.RawMessage + err error + calls int +} + +func (s *settlementStateStub) WriteBilling(_ context.Context, id string, value json.RawMessage) error { + s.calls++ + s.id = id + s.billing = append(json.RawMessage(nil), value...) + return s.err +} diff --git a/backend/internal/postgres/administration.go b/backend/internal/postgres/administration.go index 998d2f9..17162ce 100644 --- a/backend/internal/postgres/administration.go +++ b/backend/internal/postgres/administration.go @@ -13,7 +13,20 @@ const organizationColumns = `id, name, status, archive_owner_id, created_at, upd const ListAdministrationAccountsSQL = `SELECT ` + accountColumns + ` FROM public.platform_users WHERE ($1::text = '' OR organization_id = $1::text) AND ($2::text = '' OR role = $2::text) AND ($3::boolean OR status = 'active') ORDER BY created_at DESC` const GetAdministrationAccountSQL = `SELECT ` + accountColumns + ` FROM public.platform_users WHERE id = $1::text` const CreateAdministrationAccountSQL = `INSERT INTO public.platform_users (` + accountColumns + `) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING ` + accountColumns -const UpdateAdministrationAccountSQL = `UPDATE public.platform_users SET phone=$2, display_name=$3, role=$4, organization_id=$5, status=$6, password_hash=$7, password_salt=$8, failed_login_count=$9, locked_until=$10, session_version=$11, last_login_at=$12, legacy_subject=$13, created_at=$14, updated_at=$15 WHERE id=$1::text RETURNING ` + accountColumns +const ApplyAdministrationAccountUpdateSQL = `UPDATE public.platform_users SET +display_name = CASE WHEN $2::boolean THEN $3::text ELSE display_name END, +role = CASE WHEN $4::boolean THEN $5::text ELSE role END, +organization_id = CASE WHEN $6::boolean THEN $7::text ELSE organization_id END, +status = CASE WHEN $8::boolean THEN $9::text ELSE status END, +password_hash = CASE WHEN $10::boolean THEN $11::text ELSE password_hash END, +password_salt = CASE WHEN $10::boolean THEN $12::text ELSE password_salt END, +failed_login_count = CASE WHEN $13::boolean THEN 0 ELSE failed_login_count END, +locked_until = CASE WHEN $13::boolean THEN NULL ELSE locked_until END, +session_version = session_version + CASE WHEN $14::boolean THEN 1 ELSE 0 END, +updated_at = $15::timestamptz +WHERE id = $1::text + AND ($16::text = 'super_admin' OR (role = 'user' AND organization_id = $17::text)) +RETURNING ` + accountColumns const ListAdministrationOrganizationsSQL = `SELECT ` + organizationColumns + ` FROM public.platform_organizations WHERE ($1::boolean OR status = 'active') ORDER BY created_at ASC` const GetAdministrationOrganizationSQL = `SELECT ` + organizationColumns + ` FROM public.platform_organizations WHERE id = $1::text` const CreateAdministrationOrganizationSQL = `INSERT INTO public.platform_organizations (` + organizationColumns + `) VALUES ($1,$2,$3,$4,$5,$6) RETURNING ` + organizationColumns @@ -24,6 +37,7 @@ const ArchiveAssetsSQL = `UPDATE public.assets SET owner_id = $2::text WHERE own const ArchiveGenerationJobsSQL = `UPDATE public.generation_jobs SET owner_id = $2::text WHERE owner_id = $1::text` const ArchiveProjectsSQL = `UPDATE public.projects SET owner_id = $2::text WHERE owner_id = $1::text` const ArchiveImageTemplatesSQL = `UPDATE public.image_templates SET owner_id = $2::text WHERE owner_id = $1::text` +const ArchiveUsageEventsSQL = `UPDATE public.usage_events SET owner_id = $2::text WHERE owner_id = $1::text` const DeleteAdministrationAccountSQL = `DELETE FROM public.platform_users WHERE id = $1::text` func (db *Database) administrationQuerier() (Querier, error) { @@ -71,8 +85,52 @@ func (db *Database) GetAccount(ctx context.Context, id string) (administration.A func (db *Database) CreateAccount(ctx context.Context, a administration.Account) (administration.Account, error) { return db.writeAccount(ctx, CreateAdministrationAccountSQL, a) } -func (db *Database) UpdateAccount(ctx context.Context, a administration.Account) (administration.Account, error) { - return db.writeAccount(ctx, UpdateAdministrationAccountSQL, a) +func (db *Database) UpdateAccount(context.Context, administration.Account) (administration.Account, error) { + return administration.Account{}, fmt.Errorf("full-row account updates are unsafe; use ApplyAccountUpdate") +} +func (db *Database) ApplyAccountUpdate(ctx context.Context, id string, update administration.AccountUpdate) (administration.Account, error) { + q, err := db.administrationQuerier() + if err != nil { + return administration.Account{}, err + } + displayName, hasDisplayName := optionalUpdateValue(update.DisplayName) + role, hasRole := optionalUpdateValue(update.Role) + organizationID, hasOrganizationID := optionalUpdateValue(update.OrganizationID) + status, hasStatus := optionalUpdateValue(update.Status) + passwordHash, passwordSalt, hasPassword := "", "", update.PasswordHash != nil + if hasPassword { + passwordHash, passwordSalt = update.PasswordHash.Hash, update.PasswordHash.Salt + } + rows, err := q.Query(ctx, ApplyAdministrationAccountUpdateSQL, + id, + hasDisplayName, displayName, + hasRole, role, + hasOrganizationID, optionalDatabaseText(organizationID), + hasStatus, status, + hasPassword, passwordHash, passwordSalt, + update.ClearLoginLock, update.IncrementSessionVersion, + update.UpdatedAt, + update.Actor.Role, update.Actor.OrganizationID, + ) + if err != nil { + return administration.Account{}, administrationWriteError(err) + } + defer rows.Close() + if !rows.Next() { + if err := rows.Err(); err != nil { + return administration.Account{}, err + } + return administration.Account{}, &administration.Error{Kind: administration.ErrorNotFound, Message: "账号不存在或无权操作。"} + } + return scanAdministrationAccount(rows) +} + +func optionalUpdateValue[T any](value *T) (T, bool) { + if value == nil { + var zero T + return zero, false + } + return *value, true } func (db *Database) writeAccount(ctx context.Context, statement string, a administration.Account) (administration.Account, error) { q, e := db.administrationQuerier() @@ -231,7 +289,7 @@ func (db *Database) DeleteAccount(ctx context.Context, id, archive string) error _ = tx.Rollback(ctx) } }() - for _, statement := range []string{ArchiveAssetsSQL, ArchiveGenerationJobsSQL, ArchiveProjectsSQL, ArchiveImageTemplatesSQL} { + for _, statement := range []string{ArchiveAssetsSQL, ArchiveGenerationJobsSQL, ArchiveProjectsSQL, ArchiveImageTemplatesSQL, ArchiveUsageEventsSQL} { if e = tx.Exec(ctx, statement, id, archive); e != nil { return fmt.Errorf("archive account ownership: %w", e) } diff --git a/backend/internal/postgres/administration_test.go b/backend/internal/postgres/administration_test.go index 61dd6c7..db9834a 100644 --- a/backend/internal/postgres/administration_test.go +++ b/backend/internal/postgres/administration_test.go @@ -2,13 +2,99 @@ package postgres import ( "context" + "database/sql" "errors" "reflect" + "strings" "testing" + "time" "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration" ) +func TestApplyAccountUpdateUsesNarrowAtomicSQLAndDatabaseVersionIncrement(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + rows := &administrationRows{rows: [][]any{{ + "user-1", "13800138000", "New Name", "user", "org-1", "active", + "new-password-hash", "new-password-salt", 3, now.Add(time.Hour), 9, now.Add(-time.Hour), + nil, now.Add(-24 * time.Hour), now, + }}} + querier := &administrationQuerier{rows: rows} + db := NewDatabase(Config{Backend: BackendPostgres}, querier) + name := "New Name" + + got, err := db.ApplyAccountUpdate(context.Background(), "user-1", administration.AccountUpdate{ + Actor: administration.Actor{ID: "super", Role: administration.RoleSuperAdmin}, + DisplayName: &name, + UpdatedAt: now, + }) + if err != nil { + t.Fatal(err) + } + if got.SessionVersion != 9 || got.PasswordHash != "new-password-hash" || got.FailedLoginCount != 3 { + t.Fatalf("updated account=%#v, want database-owned security fields preserved", got) + } + if querier.sql != ApplyAdministrationAccountUpdateSQL { + t.Fatalf("SQL=%q", querier.sql) + } + for _, forbidden := range []string{"phone=$", "created_at=$", "last_login_at=$", "legacy_subject=$"} { + if strings.Contains(querier.sql, forbidden) { + t.Fatalf("atomic PATCH must not overwrite unrelated column: SQL contains %q", forbidden) + } + } + for _, required := range []string{ + "password_hash = CASE WHEN $10::boolean THEN $11::text ELSE password_hash END", + "failed_login_count = CASE WHEN $13::boolean THEN 0 ELSE failed_login_count END", + "session_version = session_version + CASE WHEN $14::boolean THEN 1 ELSE 0 END", + } { + if !strings.Contains(querier.sql, required) { + t.Fatalf("atomic PATCH SQL missing %q", required) + } + } + wantArgs := []any{ + "user-1", + true, "New Name", + false, administration.Role(""), + false, nil, + false, administration.Status(""), + false, "", "", + false, false, + now, + administration.RoleSuperAdmin, "", + } + if !reflect.DeepEqual(querier.args, wantArgs) { + t.Fatalf("args=%#v want %#v", querier.args, wantArgs) + } +} + +func TestApplyAccountUpdateAtomicallyResetsPasswordClearsLockAndIncrementsCurrentVersion(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + rows := &administrationRows{rows: [][]any{{ + "user-1", "13800138000", "Name", "user", "org-1", "active", + "replacement-hash", "replacement-salt", 0, nil, 12, nil, + nil, now.Add(-24 * time.Hour), now, + }}} + querier := &administrationQuerier{rows: rows} + db := NewDatabase(Config{Backend: BackendPostgres}, querier) + + got, err := db.ApplyAccountUpdate(context.Background(), "user-1", administration.AccountUpdate{ + Actor: administration.Actor{Role: administration.RoleOrganizationAdmin, OrganizationID: "org-1"}, + PasswordHash: &administration.PasswordHash{Hash: "replacement-hash", Salt: "replacement-salt"}, + ClearLoginLock: true, + IncrementSessionVersion: true, + UpdatedAt: now, + }) + if err != nil { + t.Fatal(err) + } + if got.SessionVersion != 12 || got.FailedLoginCount != 0 || got.LockedUntil != nil { + t.Fatalf("updated account=%#v", got) + } + if !reflect.DeepEqual(querier.args[9:14], []any{true, "replacement-hash", "replacement-salt", true, true}) { + t.Fatalf("security args=%#v", querier.args[9:14]) + } +} + func TestAdministrationListAccountsUsesExplicitColumnsAndParameterizedFilters(t *testing.T) { rows := &identityRows{} querier := &identityQuerier{rows: rows} @@ -31,7 +117,7 @@ func TestDeleteAccountArchivesAllOwnedDataAndIdentityInOneTransaction(t *testing if err := db.DeleteAccount(context.Background(), "user-1", "archive:org-1"); err != nil { t.Fatal(err) } - wantSQL := []string{ArchiveAssetsSQL, ArchiveGenerationJobsSQL, ArchiveProjectsSQL, ArchiveImageTemplatesSQL, DeleteAdministrationAccountSQL} + wantSQL := []string{ArchiveAssetsSQL, ArchiveGenerationJobsSQL, ArchiveProjectsSQL, ArchiveImageTemplatesSQL, ArchiveUsageEventsSQL, DeleteAdministrationAccountSQL} if !reflect.DeepEqual(tx.sql, wantSQL) { t.Fatalf("SQL sequence=%#v want %#v", tx.sql, wantSQL) } @@ -94,3 +180,56 @@ func (t *administrationTransaction) Exec(_ context.Context, query string, args . } func (t *administrationTransaction) Commit(context.Context) error { t.commits++; return nil } func (t *administrationTransaction) Rollback(context.Context) error { t.rollbacks++; return nil } + +type administrationQuerier struct { + rows *administrationRows + sql string + args []any +} + +func (q *administrationQuerier) Query(_ context.Context, query string, args ...any) (Rows, error) { + q.sql, q.args = query, args + return q.rows, nil +} + +type administrationRows struct { + rows [][]any + idx int +} + +func (r *administrationRows) Close() {} +func (r *administrationRows) Err() error { return nil } +func (r *administrationRows) Next() bool { return r.idx < len(r.rows) } +func (r *administrationRows) Scan(dest ...any) error { + if r.idx >= len(r.rows) || len(dest) != len(r.rows[r.idx]) { + return errors.New("invalid administration row scan") + } + row := r.rows[r.idx] + r.idx++ + for i, target := range dest { + value := row[i] + switch target := target.(type) { + case *string: + *target = value.(string) + case *int: + *target = value.(int) + case *administration.Role: + *target = administration.Role(value.(string)) + case *administration.Status: + *target = administration.Status(value.(string)) + case *time.Time: + *target = value.(time.Time) + case *sql.NullString: + if value != nil { + *target = sql.NullString{String: value.(string), Valid: true} + } + case *sql.NullTime: + if value != nil { + *target = sql.NullTime{Time: value.(time.Time), Valid: true} + } + default: + return errors.New("unsupported administration scan target") + } + } + return nil +} diff --git a/backend/internal/postgres/assets.go b/backend/internal/postgres/assets.go index 08ec2e9..dc93383 100644 --- a/backend/internal/postgres/assets.go +++ b/backend/internal/postgres/assets.go @@ -31,6 +31,10 @@ const GetOwnerAssetSQL = `SELECT ` + assetFields + ` FROM public.assets AS a WHERE a.owner_id = $1::text AND a.id = $2::text LIMIT 1` +const GetOwnerAssetByStoragePathSQL = `SELECT ` + assetFields + ` +FROM public.assets AS a +WHERE a.owner_id = $1::text AND a.storage_path = $2::text +LIMIT 1` const listPublicAccessibleAssetIDs = `SELECT unnest(j.input_asset_ids || j.output_asset_ids) AS asset_id FROM ( SELECT input_asset_ids, output_asset_ids @@ -72,6 +76,9 @@ func (db *Database) ListOwner(ctx context.Context, owner string) ([]assets.Asset func (db *Database) GetOwner(ctx context.Context, owner, id string) (assets.Asset, bool, error) { return db.oneAsset(ctx, GetOwnerAssetSQL, owner, id) } +func (db *Database) GetOwnerByStoragePath(ctx context.Context, owner, storagePath string) (assets.Asset, bool, error) { + return db.oneAsset(ctx, GetOwnerAssetByStoragePathSQL, owner, storagePath) +} func (db *Database) ListPublic(ctx context.Context, owner, client string, limit int) ([]assets.Asset, error) { return db.listAssets(ctx, ListPublicAssetsSQL, owner, assets.ClientTag(client), client, limit) } diff --git a/backend/internal/postgres/assets_test.go b/backend/internal/postgres/assets_test.go index bda5421..5faa902 100644 --- a/backend/internal/postgres/assets_test.go +++ b/backend/internal/postgres/assets_test.go @@ -24,6 +24,9 @@ func TestAssetCatalogUsesExplicitOwnerScopedQueries(t *testing.T) { {name: "get owner", call: func(db *Database) (assets.Asset, bool, error) { return db.GetOwner(context.Background(), "owner-1", "asset-1") }, wantSQL: GetOwnerAssetSQL, wantArgs: []any{"owner-1", "asset-1"}}, + {name: "get storage path within owner", call: func(db *Database) (assets.Asset, bool, error) { + return db.GetOwnerByStoragePath(context.Background(), "owner-1", "uploads/a.png") + }, wantSQL: GetOwnerAssetByStoragePathSQL, wantArgs: []any{"owner-1", "uploads/a.png"}}, {name: "get public", call: func(db *Database) (assets.Asset, bool, error) { return db.GetPublic(context.Background(), "api:agent-a", "agent-a", "asset-1", 200) }, wantSQL: GetPublicAssetSQL, wantArgs: []any{"api:agent-a", "asset-1", "api-client:agent-a", "agent-a", 200}}, diff --git a/backend/internal/postgres/billing.go b/backend/internal/postgres/billing.go index 572f114..a5fa79d 100644 --- a/backend/internal/postgres/billing.go +++ b/backend/internal/postgres/billing.go @@ -9,11 +9,20 @@ import ( "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" ) -const ListBillingPriceRulesSQL = `SELECT id, provider, capability, req_key, unit, standard_unit_price_fen, markup_multiplier, enabled, conditions, quantity_source, priority, parameter_dimensions +const ListBillingPriceRulesSQL = `SELECT id, provider, capability, req_key, variant_key, unit, standard_unit_price_fen, markup_multiplier, enabled, conditions, quantity_source, priority, note, source, parameter_dimensions, created_at::text, updated_at::text FROM public.billing_price_rules WHERE ($1::boolean OR enabled = true) ORDER BY provider, capability, id` +const GetBillingPriceRuleSQL = `SELECT id, provider, capability, req_key, variant_key, unit, standard_unit_price_fen, markup_multiplier, enabled, conditions, quantity_source, priority, note, source, parameter_dimensions, created_at::text, updated_at::text FROM public.billing_price_rules WHERE id = $1::text LIMIT 1` +const UpdateBillingPriceRuleSQL = `UPDATE public.billing_price_rules SET markup_multiplier = CASE WHEN $2::text = '' THEN $3::numeric ELSE markup_multiplier END, parameter_dimensions = CASE WHEN $2::text = '' THEN parameter_dimensions ELSE (SELECT jsonb_agg(CASE WHEN dimension->>'key' = $2::text THEN jsonb_set(dimension, '{tiers}', (SELECT jsonb_agg(CASE WHEN tier->>'value' = $4::text THEN jsonb_set(tier, '{markupMultiplier}', to_jsonb($3::numeric), true) ELSE tier END) FROM jsonb_array_elements(dimension->'tiers') tier), true) ELSE dimension END) FROM jsonb_array_elements(parameter_dimensions) dimension) END, updated_at = now() WHERE id = $1::text RETURNING id, provider, capability, req_key, variant_key, unit, standard_unit_price_fen, markup_multiplier, enabled, conditions, quantity_source, priority, note, source, parameter_dimensions, created_at::text, updated_at::text` +const GetBillingWalletSQL = `SELECT $1::text, COALESCE(balance_fen, 0), COALESCE(total_recharged_fen, 0), COALESCE(total_charged_fen, 0), COALESCE(updated_at::text, '') FROM public.billing_wallets WHERE organization_id = $1::text UNION ALL SELECT $1::text, 0, 0, 0, '' WHERE NOT EXISTS (SELECT 1 FROM public.billing_wallets WHERE organization_id = $1::text) LIMIT 1` +const ListBillingWalletsSQL = `SELECT organization_id, balance_fen, total_recharged_fen, total_charged_fen, updated_at::text FROM public.billing_wallets ORDER BY updated_at DESC` +const ListBillingLedgerSQL = `SELECT id, organization_id, COALESCE(account_id, ''), COALESCE(job_id, ''), kind, delta_fen, balance_after_fen, currency, idempotency_key, description, metadata, created_at::text FROM public.billing_ledger WHERE ($1::text = '' OR organization_id = $1::text) AND ($2::text = '' OR account_id = $2::text) ORDER BY created_at DESC LIMIT $3::integer` +const ListBillingOrganizationsSQL = `SELECT id, name, status, archive_owner_id, created_at::text, updated_at::text FROM public.platform_organizations ORDER BY created_at ASC` +const ListBillingMembersSQL = `SELECT id, display_name, phone, role, COALESCE(organization_id, ''), status FROM public.platform_users ORDER BY created_at ASC` +const BillingOrganizationExistsSQL = `SELECT EXISTS (SELECT 1 FROM public.platform_organizations WHERE id = $1::text)` + type BillingWalletPoster struct{ database *Database } func NewBillingWalletPoster(database *Database) BillingWalletPoster { @@ -30,6 +39,9 @@ func (poster BillingWalletPoster) PostWalletEntry(ctx context.Context, p billing } return billing.WalletPosting{LedgerID: row.LedgerID, BalanceAfterFen: row.BalanceAfterFen, BalanceFen: row.BalanceFen, TotalRechargedFen: row.TotalRechargedFen, TotalChargedFen: row.TotalChargedFen, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt, DeltaFen: row.DeltaFen}, nil } +func (db *Database) PostBillingWalletEntry(ctx context.Context, p billing.WalletPostParams) (billing.WalletPosting, error) { + return NewBillingWalletPoster(db).PostWalletEntry(ctx, p) +} func (db *Database) ListBillingPriceRules(ctx context.Context, includeDisabled bool) ([]billing.PriceRule, error) { if db.config.Backend != BackendPostgres || db.querier == nil { @@ -42,27 +54,181 @@ func (db *Database) ListBillingPriceRules(ctx context.Context, includeDisabled b defer rows.Close() var out []billing.PriceRule for rows.Next() { - var rule billing.PriceRule - var req, quantity sql.NullString - var conditions, dimensions json.RawMessage - if err := rows.Scan(&rule.ID, &rule.Provider, &rule.Capability, &req, &rule.Unit, &rule.StandardUnitPriceFen, &rule.MarkupMultiplier, &rule.Enabled, &conditions, &quantity, &rule.Priority, &dimensions); err != nil { - return nil, fmt.Errorf("scan billing price rule: %w", err) - } - rule.ReqKey = req.String - rule.QuantitySource = billing.QuantitySource(quantity.String) - if len(conditions) > 0 { - if err := json.Unmarshal(conditions, &rule.Conditions); err != nil { - return nil, fmt.Errorf("decode billing conditions: %w", err) - } - } - if len(dimensions) > 0 { - if err := json.Unmarshal(dimensions, &rule.Dimensions); err != nil { - return nil, fmt.Errorf("decode billing dimensions: %w", err) - } + rule, err := scanFullPriceRule(rows) + if err != nil { + return nil, err } out = append(out, rule) } return out, rows.Err() } +func (db *Database) GetBillingPriceRule(ctx context.Context, id string) (*billing.PriceRule, error) { + rows, err := db.billingQuery(ctx, GetBillingPriceRuleSQL, id) + if err != nil { + return nil, err + } + defer rows.Close() + if !rows.Next() { + return nil, rows.Err() + } + rule, err := scanFullPriceRule(rows) + return &rule, err +} +func (db *Database) UpdateBillingPriceRule(ctx context.Context, id string, patch billing.PricePatch) (*billing.PriceRule, error) { + rows, err := db.billingQuery(ctx, UpdateBillingPriceRuleSQL, id, patch.DimensionKey, patch.MarkupMultiplier, patch.TierValue) + if err != nil { + return nil, err + } + defer rows.Close() + if !rows.Next() { + return nil, rows.Err() + } + rule, err := scanFullPriceRule(rows) + return &rule, err +} +func scanFullPriceRule(rows Rows) (billing.PriceRule, error) { + var rule billing.PriceRule + var req, variant, quantity, note sql.NullString + var conditions, source, dimensions json.RawMessage + if err := rows.Scan(&rule.ID, &rule.Provider, &rule.Capability, &req, &variant, &rule.Unit, &rule.StandardUnitPriceFen, &rule.MarkupMultiplier, &rule.Enabled, &conditions, &quantity, &rule.Priority, ¬e, &source, &dimensions, &rule.CreatedAt, &rule.UpdatedAt); err != nil { + return rule, fmt.Errorf("scan billing price rule: %w", err) + } + rule.ReqKey, rule.VariantKey, rule.QuantitySource, rule.Note = req.String, variant.String, billing.QuantitySource(quantity.String), note.String + if len(conditions) > 0 { + if err := json.Unmarshal(conditions, &rule.Conditions); err != nil { + return rule, err + } + } + if len(source) > 0 && string(source) != "null" { + if err := json.Unmarshal(source, &rule.Source); err != nil { + return rule, err + } + } + if len(dimensions) > 0 { + if err := json.Unmarshal(dimensions, &rule.Dimensions); err != nil { + return rule, err + } + } + return rule, nil +} +func (db *Database) BillingWallet(ctx context.Context, organizationID string) (billing.Wallet, error) { + rows, err := db.billingQuery(ctx, GetBillingWalletSQL, organizationID) + if err != nil { + return billing.Wallet{}, err + } + defer rows.Close() + if !rows.Next() { + return billing.Wallet{}, rows.Err() + } + return scanBillingWallet(rows) +} +func (db *Database) BillingWallets(ctx context.Context) ([]billing.Wallet, error) { + rows, err := db.billingQuery(ctx, ListBillingWalletsSQL) + if err != nil { + return nil, err + } + defer rows.Close() + var out []billing.Wallet + for rows.Next() { + wallet, err := scanBillingWallet(rows) + if err != nil { + return nil, err + } + out = append(out, wallet) + } + return out, rows.Err() +} +func scanBillingWallet(rows Rows) (billing.Wallet, error) { + var wallet billing.Wallet + if err := rows.Scan(&wallet.OrganizationID, &wallet.BalanceFen, &wallet.TotalRechargedFen, &wallet.TotalChargedFen, &wallet.UpdatedAt); err != nil { + return wallet, fmt.Errorf("scan billing wallet: %w", err) + } + wallet.Currency = billing.CurrencyCNY + return wallet, nil +} +func (db *Database) BillingLedger(ctx context.Context, organizationID, accountID string, limit int) ([]billing.LedgerEntry, error) { + if limit <= 0 || limit > 500 { + limit = 500 + } + rows, err := db.billingQuery(ctx, ListBillingLedgerSQL, organizationID, accountID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []billing.LedgerEntry + for rows.Next() { + var entry billing.LedgerEntry + var metadata json.RawMessage + if err := rows.Scan(&entry.ID, &entry.OrganizationID, &entry.AccountID, &entry.JobID, &entry.Kind, &entry.DeltaFen, &entry.BalanceAfterFen, &entry.Currency, &entry.IdempotencyKey, &entry.Description, &metadata, &entry.CreatedAt); err != nil { + return nil, err + } + if len(metadata) > 0 { + if err := json.Unmarshal(metadata, &entry.Metadata); err != nil { + return nil, err + } + } + out = append(out, entry) + } + return out, rows.Err() +} +func (db *Database) BillingOrganizations(ctx context.Context) ([]billing.Organization, error) { + rows, err := db.billingQuery(ctx, ListBillingOrganizationsSQL) + if err != nil { + return nil, err + } + defer rows.Close() + var out []billing.Organization + for rows.Next() { + var item billing.Organization + if err := rows.Scan(&item.ID, &item.Name, &item.Status, &item.ArchiveOwnerID, &item.CreatedAt, &item.UpdatedAt); err != nil { + return nil, err + } + out = append(out, item) + } + return out, rows.Err() +} +func (db *Database) BillingMembers(ctx context.Context) ([]billing.Member, error) { + rows, err := db.billingQuery(ctx, ListBillingMembersSQL) + if err != nil { + return nil, err + } + defer rows.Close() + var out []billing.Member + for rows.Next() { + var item billing.Member + if err := rows.Scan(&item.ID, &item.DisplayName, &item.Phone, &item.Role, &item.OrganizationID, &item.Status); err != nil { + return nil, err + } + out = append(out, item) + } + return out, rows.Err() +} +func (db *Database) BillingOrganizationExists(ctx context.Context, id string) (bool, error) { + rows, err := db.billingQuery(ctx, BillingOrganizationExistsSQL, id) + if err != nil { + return false, err + } + defer rows.Close() + if !rows.Next() { + return false, rows.Err() + } + var exists bool + if err := rows.Scan(&exists); err != nil { + return false, err + } + return exists, rows.Err() +} +func (db *Database) billingQuery(ctx context.Context, query string, args ...any) (Rows, error) { + if db.config.Backend != BackendPostgres || db.querier == nil { + return nil, fmt.Errorf("PostgreSQL is unavailable when ZHINIAN_DATA_BACKEND=%s", db.config.Backend) + } + rows, err := db.querier.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("billing query: %w", err) + } + return rows, nil +} + var _ billing.WalletPoster = BillingWalletPoster{} +var _ billing.Store = (*Database)(nil) diff --git a/backend/internal/postgres/billing_defaults.go b/backend/internal/postgres/billing_defaults.go new file mode 100644 index 0000000..88e77c2 --- /dev/null +++ b/backend/internal/postgres/billing_defaults.go @@ -0,0 +1,38 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" +) + +const SeedBillingPriceRulesSQL = `INSERT INTO public.billing_price_rules (id, provider, capability, req_key, variant_key, unit, standard_unit_price_fen, markup_multiplier, enabled, conditions, quantity_source, priority, note, source, parameter_dimensions) +SELECT rule->>'id', rule->>'provider', rule->>'capability', NULLIF(rule->>'reqKey', ''), NULLIF(rule->>'variantKey', ''), rule->>'unit', (rule->>'standardUnitPriceFen')::bigint, (rule->>'markupMultiplier')::numeric, COALESCE((rule->>'enabled')::boolean, true), COALESCE(rule->'conditions', '{}'::jsonb), NULLIF(rule->>'quantitySource', ''), COALESCE((rule->>'priority')::integer, 0), NULLIF(rule->>'note', ''), rule->'source', COALESCE(rule->'parameterDimensions', '[]'::jsonb) +FROM jsonb_array_elements($1::jsonb) AS rule +ON CONFLICT (provider, capability, COALESCE(req_key, ''), COALESCE(variant_key, ''), COALESCE(conditions, '{}'::jsonb)) DO UPDATE SET +req_key = EXCLUDED.req_key, variant_key = EXCLUDED.variant_key, unit = EXCLUDED.unit, standard_unit_price_fen = EXCLUDED.standard_unit_price_fen, enabled = EXCLUDED.enabled, conditions = EXCLUDED.conditions, quantity_source = EXCLUDED.quantity_source, priority = EXCLUDED.priority, note = EXCLUDED.note, source = EXCLUDED.source, parameter_dimensions = EXCLUDED.parameter_dimensions, updated_at = now()` + +func (db *Database) SeedBillingPriceRules(ctx context.Context, rules []billing.PriceRule) error { + if len(rules) == 0 { + return nil + } + if db.config.Backend != BackendPostgres || db.querier == nil { + return fmt.Errorf("PostgreSQL is unavailable when ZHINIAN_DATA_BACKEND=%s", db.config.Backend) + } + payload, err := json.Marshal(rules) + if err != nil { + return fmt.Errorf("encode billing price rule defaults: %w", err) + } + rows, err := db.querier.Query(ctx, SeedBillingPriceRulesSQL, payload) + if err != nil { + return fmt.Errorf("seed billing price rule defaults: %w", err) + } + defer rows.Close() + for rows.Next() { + } + return rows.Err() +} + +var _ billing.PriceRuleSeeder = (*Database)(nil) diff --git a/backend/internal/postgres/billing_defaults_test.go b/backend/internal/postgres/billing_defaults_test.go new file mode 100644 index 0000000..e81d8ce --- /dev/null +++ b/backend/internal/postgres/billing_defaults_test.go @@ -0,0 +1,33 @@ +package postgres + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" +) + +func TestSeedBillingPriceRulesUsesOneIdempotentUpsert(t *testing.T) { + q := &fakeQuerier{} + db := NewDatabase(Config{Backend: BackendPostgres}, q) + rules := []billing.PriceRule{{ID: "r", Provider: "seedance", Capability: "video.generate", ReqKey: "m", VariantKey: "resolution=720p", Unit: billing.UnitVideoSecond, StandardUnitPriceFen: 99, MarkupMultiplier: 1.2, Enabled: true}} + if err := db.SeedBillingPriceRules(context.Background(), rules); err != nil { + t.Fatal(err) + } + if q.sql != SeedBillingPriceRulesSQL || !strings.Contains(q.sql, "ON CONFLICT (provider, capability, COALESCE(req_key, ''), COALESCE(variant_key, ''), COALESCE(conditions, '{}'::jsonb)) DO UPDATE") { + t.Fatalf("sql = %q", q.sql) + } + if len(q.args) != 1 { + t.Fatalf("args = %#v", q.args) + } + var decoded []billing.PriceRule + if err := json.Unmarshal(q.args[0].([]byte), &decoded); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(decoded, rules) { + t.Fatalf("payload = %#v", decoded) + } +} diff --git a/backend/internal/postgres/database.go b/backend/internal/postgres/database.go index 2078c84..a666040 100644 --- a/backend/internal/postgres/database.go +++ b/backend/internal/postgres/database.go @@ -21,6 +21,12 @@ WITH required_table_privileges(table_name, privilege_name) AS ( ('billing_price_rules', 'SELECT'), ('billing_price_rules', 'INSERT'), ('billing_price_rules', 'UPDATE'), ('billing_wallets', 'SELECT'), ('billing_wallets', 'INSERT'), ('billing_wallets', 'UPDATE'), ('billing_ledger', 'SELECT'), ('billing_ledger', 'INSERT') +), +required_generation_job_columns(column_name) AS ( + VALUES + ('provider_dispatch_started_at'), + ('dispatch_ready_at'), + ('finalized_at') ) SELECT NOT EXISTS ( @@ -29,6 +35,17 @@ SELECT WHERE to_regclass('public.' || table_name) IS NULL OR NOT has_table_privilege(current_user, 'public.' || table_name, privilege_name) ) + AND NOT EXISTS ( + SELECT 1 + FROM required_generation_job_columns AS required_column + WHERE NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'generation_jobs' + AND column_name = required_column.column_name + ) + ) AND has_function_privilege( current_user, 'public.claim_generation_jobs(text,integer,integer)', diff --git a/backend/internal/postgres/database_test.go b/backend/internal/postgres/database_test.go index a271ff3..faf1b81 100644 --- a/backend/internal/postgres/database_test.go +++ b/backend/internal/postgres/database_test.go @@ -33,6 +33,28 @@ func TestReadinessSQLFreezesPrivilegeMatrixAndFunctionSignatures(t *testing.T) { } } +func TestReadinessSQLRequiresPublicGenerationJobLifecycleColumns(t *testing.T) { + for _, fragment := range []string{ + "required_generation_job_columns(column_name)", + "information_schema.columns", + "table_schema = 'public'", + "table_name = 'generation_jobs'", + } { + if !strings.Contains(ReadinessSQL, fragment) { + t.Errorf("ReadinessSQL missing public generation_jobs column check %q", fragment) + } + } + for _, column := range []string{ + "provider_dispatch_started_at", + "dispatch_ready_at", + "finalized_at", + } { + if !strings.Contains(ReadinessSQL, "('"+column+"')") { + t.Errorf("ReadinessSQL missing required lifecycle column %q", column) + } + } +} + func TestReadinessLocalSucceedsWithoutQuery(t *testing.T) { db := NewDatabase(Config{Backend: BackendLocal}, &fakeQuerier{err: errors.New("must not query")}) if err := db.Readiness(context.Background()); err != nil { diff --git a/backend/internal/postgres/generation_state.go b/backend/internal/postgres/generation_state.go new file mode 100644 index 0000000..56fc70b --- /dev/null +++ b/backend/internal/postgres/generation_state.go @@ -0,0 +1,134 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "github.com/jackc/pgx/v5/pgconn" +) + +const activateChargedCreationSQL = `UPDATE public.generation_jobs +SET billing = $2::jsonb, dispatch_ready_at = COALESCE(dispatch_ready_at, now()), updated_at = now() +WHERE id = $1::text AND (dispatch_ready_at IS NULL OR billing = $2::jsonb) RETURNING id` + +const reconcileChargedCreationSQL = `SELECT billing, dispatch_ready_at::text FROM public.generation_jobs WHERE id = $1::text LIMIT 1` + +// ChargeAndActivateCreation closes the only non-idempotent creation window: +// the wallet function and job dispatch gate commit in one database transaction. +// A deterministic ledger ID is safe because billing_post_wallet_entry already +// owns payload-idempotency validation for job-charge:. +func (db *Database) ChargeAndActivateCreation(ctx context.Context, request billing.ChargeRequest, pending json.RawMessage) (json.RawMessage, error) { + if db == nil || db.config.Backend != BackendPostgres || db.transactions == nil { + return nil, fmt.Errorf("charge and activate generation creation: PostgreSQL transaction is unavailable") + } + if request.JobID == "" || request.OrganizationID == "" || request.AmountFen <= 0 || len(pending) == 0 || !json.Valid(pending) { + return nil, fmt.Errorf("charge and activate generation creation: invalid request") + } + metadata, err := json.Marshal(request.Metadata) + if err != nil { + return nil, fmt.Errorf("charge and activate generation creation: encode metadata: %w", err) + } + tx, err := db.transactions.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("charge and activate generation creation: begin: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback(ctx) + } + }() + rows, err := tx.Query(ctx, PostWalletEntrySQL, + "job-charge-entry:"+request.JobID, request.OrganizationID, optionalDatabaseText(request.AccountID), request.JobID, + "charge", -request.AmountFen, billing.CurrencyCNY, "job-charge:"+request.JobID, request.Description, json.RawMessage(metadata), + ) + if err != nil { + return nil, fmt.Errorf("charge and activate generation creation: post wallet: %w", err) + } + var posting WalletEntry + if !rows.Next() { + rows.Close() + return nil, fmt.Errorf("charge and activate generation creation: wallet returned no row") + } + if err := rows.Scan(&posting.LedgerID, &posting.BalanceAfterFen, &posting.BalanceFen, &posting.TotalRechargedFen, &posting.TotalChargedFen, &posting.CreatedAt, &posting.UpdatedAt, &posting.DeltaFen); err != nil { + rows.Close() + return nil, fmt.Errorf("charge and activate generation creation: scan wallet: %w", err) + } + rows.Close() + var state map[string]any + if err := json.Unmarshal(pending, &state); err != nil { + return nil, fmt.Errorf("charge and activate generation creation: decode billing: %w", err) + } + state["status"] = "charged" + state["ledgerEntryId"] = posting.LedgerID + chargedAt := posting.CreatedAt + if chargedAt.IsZero() { + chargedAt = time.Now().UTC() + } + state["chargedAt"] = chargedAt.UTC().Format(time.RFC3339Nano) + charged, err := json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("charge and activate generation creation: encode billing: %w", err) + } + activation, err := tx.Query(ctx, activateChargedCreationSQL, request.JobID, json.RawMessage(charged)) + if err != nil { + return nil, fmt.Errorf("charge and activate generation creation: activate job: %w", err) + } + if !activation.Next() { + activation.Close() + return nil, fmt.Errorf("charge and activate generation creation: job is not pending") + } + var id string + if err := activation.Scan(&id); err != nil { + activation.Close() + return nil, fmt.Errorf("charge and activate generation creation: scan job: %w", err) + } + activation.Close() + commitErr := tx.Commit(ctx) + committed = true + if commitErr != nil { + return db.reconcileChargedCreation(request.JobID, commitErr) + } + return charged, nil +} + +func (db *Database) reconcileChargedCreation(jobID string, commitErr error) (json.RawMessage, error) { + queryCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + rows, err := db.querier.Query(queryCtx, reconcileChargedCreationSQL, jobID) + if err != nil { + return nil, fmt.Errorf("%w: reconcile charge and activate generation creation after commit error: %v (commit: %v)", billing.ErrCommitOutcomeUnknown, err, commitErr) + } + defer rows.Close() + if !rows.Next() { + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("%w: reconcile charge and activate generation creation after commit error: %v (commit: %v)", billing.ErrCommitOutcomeUnknown, err, commitErr) + } + return unresolvedCommitOutcome(commitErr, "job not found during reconciliation") + } + var persistedBytes []byte + var dispatchReadyAt string + if err := rows.Scan(&persistedBytes, &dispatchReadyAt); err != nil { + return nil, fmt.Errorf("%w: scan charge and activate reconciliation: %v (commit: %v)", billing.ErrCommitOutcomeUnknown, err, commitErr) + } + persisted := json.RawMessage(persistedBytes) + var state struct { + Status string `json:"status"` + } + if dispatchReadyAt == "" || json.Unmarshal(persisted, &state) != nil || state.Status != "charged" { + return unresolvedCommitOutcome(commitErr, "job was not activated during reconciliation") + } + return persisted, nil +} + +func unresolvedCommitOutcome(commitErr error, reconciliation string) (json.RawMessage, error) { + var serverError *pgconn.PgError + if errors.As(commitErr, &serverError) { + return nil, fmt.Errorf("charge and activate generation creation: PostgreSQL rejected commit (%s): %w", reconciliation, commitErr) + } + return nil, fmt.Errorf("%w: %s after commit error: %v", billing.ErrCommitOutcomeUnknown, reconciliation, commitErr) +} diff --git a/backend/internal/postgres/generation_state_test.go b/backend/internal/postgres/generation_state_test.go new file mode 100644 index 0000000..51c8104 --- /dev/null +++ b/backend/internal/postgres/generation_state_test.go @@ -0,0 +1,157 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing" + "github.com/jackc/pgx/v5/pgconn" +) + +func TestChargeAndActivateCreationCommitsWalletAndDispatchGateTogether(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + tx := &generationStateTransaction{results: []*jobRows{ + {rows: [][]any{{"ledger-1", int64(965), int64(965), int64(1000), int64(35), now, now, int64(-35)}}}, + {rows: [][]any{{"job-1"}}}, + }} + db := NewDatabase(Config{Backend: BackendPostgres}, &generationStatePool{tx: tx}) + got, err := db.ChargeAndActivateCreation(context.Background(), billing.ChargeRequest{OrganizationID: "org", AccountID: "account", JobID: "job-1", AmountFen: 35, Description: "image"}, json.RawMessage(`{"status":"pending","amountFen":35}`)) + if err != nil || tx.commits != 1 || tx.rollbacks != 0 || len(tx.queries) != 2 { + t.Fatalf("got=%s err=%v commits=%d rollbacks=%d queries=%d", got, err, tx.commits, tx.rollbacks, len(tx.queries)) + } + if tx.queries[0].sql != PostWalletEntrySQL || tx.queries[1].sql != activateChargedCreationSQL { + t.Fatalf("queries=%#v", tx.queries) + } + if tx.queries[0].args[7] != "job-charge:job-1" || tx.queries[1].args[0] != "job-1" { + t.Fatalf("args=%#v", tx.queries) + } + var state map[string]any + _ = json.Unmarshal(got, &state) + if state["status"] != "charged" || state["ledgerEntryId"] != "ledger-1" { + t.Fatalf("billing=%#v", state) + } +} + +func TestChargeAndActivateCreationRollsBackWhenActivationFails(t *testing.T) { + now := time.Now() + tx := &generationStateTransaction{results: []*jobRows{ + {rows: [][]any{{"ledger-1", int64(1), int64(1), int64(1), int64(1), now, now, int64(-1)}}}, + {}, + }} + db := NewDatabase(Config{Backend: BackendPostgres}, &generationStatePool{tx: tx}) + _, err := db.ChargeAndActivateCreation(context.Background(), billing.ChargeRequest{OrganizationID: "org", JobID: "job", AmountFen: 1}, json.RawMessage(`{"status":"pending"}`)) + if err == nil || tx.commits != 0 || tx.rollbacks != 1 { + t.Fatalf("err=%v commits=%d rollbacks=%d", err, tx.commits, tx.rollbacks) + } +} + +func TestChargeAndActivateCreationTreatsCommittedButLostResponseAsSuccess(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + charged := json.RawMessage(`{"status":"charged","amountFen":35,"ledgerEntryId":"ledger-1"}`) + tx := &generationStateTransaction{ + results: []*jobRows{ + {rows: [][]any{{"ledger-1", int64(965), int64(965), int64(1000), int64(35), now, now, int64(-35)}}}, + {rows: [][]any{{"job-1"}}}, + }, + commitErr: errors.New("connection lost while reading COMMIT response"), + } + pool := &generationStatePool{tx: tx, rows: &jobRows{rows: [][]any{{[]byte(charged), now.Format(time.RFC3339Nano)}}}} + db := NewDatabase(Config{Backend: BackendPostgres}, pool) + + got, err := db.ChargeAndActivateCreation(context.Background(), billing.ChargeRequest{OrganizationID: "org", JobID: "job-1", AmountFen: 35}, json.RawMessage(`{"status":"pending","amountFen":35}`)) + if err != nil || string(got) != string(charged) || pool.queries != 1 || tx.rollbacks != 0 { + t.Fatalf("got=%s err=%v reconciliation queries=%d rollbacks=%d", got, err, pool.queries, tx.rollbacks) + } +} + +func TestChargeAndActivateCreationReturnsDefiniteCommitFailureWhenJobWasNotActivated(t *testing.T) { + tx := committedGenerationStateTransaction(&pgconn.PgError{Code: "40001", Message: "could not serialize access due to concurrent update"}) + pool := &generationStatePool{tx: tx, rows: &jobRows{}} + db := NewDatabase(Config{Backend: BackendPostgres}, pool) + + _, err := db.ChargeAndActivateCreation(context.Background(), billing.ChargeRequest{OrganizationID: "org", JobID: "job-1", AmountFen: 35}, json.RawMessage(`{"status":"pending"}`)) + if err == nil || errors.Is(err, billing.ErrCommitOutcomeUnknown) { + t.Fatalf("err=%v, want definite ordinary commit failure", err) + } +} + +func TestChargeAndActivateCreationReturnsUnknownOutcomeWhenReconciliationStillShowsPending(t *testing.T) { + tx := committedGenerationStateTransaction(errors.New("commit response unavailable")) + pool := &generationStatePool{tx: tx, rows: &jobRows{rows: [][]any{{[]byte(`{"status":"pending","amountFen":35}`), ""}}}} + db := NewDatabase(Config{Backend: BackendPostgres}, pool) + + _, err := db.ChargeAndActivateCreation(context.Background(), billing.ChargeRequest{OrganizationID: "org", JobID: "job-1", AmountFen: 35}, json.RawMessage(`{"status":"pending"}`)) + if !errors.Is(err, billing.ErrCommitOutcomeUnknown) { + t.Fatalf("err=%v, want ErrCommitOutcomeUnknown", err) + } +} + +func TestChargeAndActivateCreationReturnsUnknownOutcomeWhenReconciliationFindsNoJob(t *testing.T) { + tx := committedGenerationStateTransaction(errors.New("commit response unavailable")) + pool := &generationStatePool{tx: tx, rows: &jobRows{}} + db := NewDatabase(Config{Backend: BackendPostgres}, pool) + + _, err := db.ChargeAndActivateCreation(context.Background(), billing.ChargeRequest{OrganizationID: "org", JobID: "job-1", AmountFen: 35}, json.RawMessage(`{"status":"pending"}`)) + if !errors.Is(err, billing.ErrCommitOutcomeUnknown) { + t.Fatalf("err=%v, want ErrCommitOutcomeUnknown", err) + } +} + +func TestChargeAndActivateCreationReturnsUnknownOutcomeWhenReconciliationFails(t *testing.T) { + tx := committedGenerationStateTransaction(errors.New("commit response unavailable")) + pool := &generationStatePool{tx: tx, err: errors.New("reconciliation connection unavailable")} + db := NewDatabase(Config{Backend: BackendPostgres}, pool) + + _, err := db.ChargeAndActivateCreation(context.Background(), billing.ChargeRequest{OrganizationID: "org", JobID: "job-1", AmountFen: 35}, json.RawMessage(`{"status":"pending"}`)) + if !errors.Is(err, billing.ErrCommitOutcomeUnknown) { + t.Fatalf("err=%v, want ErrCommitOutcomeUnknown", err) + } +} + +func committedGenerationStateTransaction(commitErr error) *generationStateTransaction { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + return &generationStateTransaction{ + results: []*jobRows{ + {rows: [][]any{{"ledger-1", int64(965), int64(965), int64(1000), int64(35), now, now, int64(-35)}}}, + {rows: [][]any{{"job-1"}}}, + }, + commitErr: commitErr, + } +} + +type generationStatePool struct { + tx *generationStateTransaction + rows *jobRows + err error + queries int +} + +func (p *generationStatePool) Query(context.Context, string, ...any) (Rows, error) { + p.queries++ + return p.rows, p.err +} +func (p *generationStatePool) Begin(context.Context) (Transaction, error) { return p.tx, nil } + +type generationStateQuery struct { + sql string + args []any +} +type generationStateTransaction struct { + results []*jobRows + queries []generationStateQuery + commits, rollbacks int + commitErr error +} + +func (t *generationStateTransaction) Query(_ context.Context, sql string, args ...any) (Rows, error) { + t.queries = append(t.queries, generationStateQuery{sql: sql, args: append([]any(nil), args...)}) + result := t.results[0] + t.results = t.results[1:] + return result, nil +} +func (*generationStateTransaction) Exec(context.Context, string, ...any) error { return nil } +func (t *generationStateTransaction) Commit(context.Context) error { t.commits++; return t.commitErr } +func (t *generationStateTransaction) Rollback(context.Context) error { t.rollbacks++; return nil } diff --git a/backend/internal/postgres/jobs.go b/backend/internal/postgres/jobs.go index 698b2f4..07c1c36 100644 --- a/backend/internal/postgres/jobs.go +++ b/backend/internal/postgres/jobs.go @@ -12,12 +12,23 @@ import ( ) const jobColumns = `id, owner_id, external_client_id, capability, provider, req_key, status, prompt, -input_asset_ids, input_urls, output_asset_ids, provider_task_id, request_payload, response_payload, +input_asset_ids, input_urls, output_asset_ids, provider_task_id, provider_dispatch_started_at, request_payload, response_payload, error, retry_of, idempotency_key, idempotency_fingerprint, priority, attempts, max_attempts, scheduled_at, locked_at, locked_by, started_at, completed_at, webhook_url, webhook_attempts, -webhook_last_status, usage_context, billing, created_at, updated_at` +webhook_last_status, usage_context, billing, dispatch_ready_at, finalized_at, created_at, updated_at` const ClaimJobsSQL = `SELECT ` + jobColumns + ` FROM public.claim_generation_jobs($1::text, $2::integer, $3::integer)` +const WriteJobBillingSQL = `UPDATE public.generation_jobs SET billing = $2::jsonb, updated_at = now() WHERE id = $1::text RETURNING id` +const ActivateJobCreationSQL = `UPDATE public.generation_jobs +SET billing = $2::jsonb, dispatch_ready_at = COALESCE(dispatch_ready_at, now()), updated_at = now() +WHERE id = $1::text AND (dispatch_ready_at IS NULL OR billing = $2::jsonb) RETURNING id` +const WriteJobOutputAssetIDsSQL = `UPDATE public.generation_jobs SET output_asset_ids = $2::text[], updated_at = now() WHERE id = $1::text RETURNING id` +const WriteJobBillingFencedSQL = `UPDATE public.generation_jobs SET billing = $2::jsonb, updated_at = now() WHERE id = $1::text AND status = $3::text AND locked_by = $4::text RETURNING id` +const WriteJobOutputAssetIDsFencedSQL = `UPDATE public.generation_jobs SET output_asset_ids = $2::text[], updated_at = now() WHERE id = $1::text AND status = $3::text AND locked_by = $4::text RETURNING id` +const FailJobCreationSQL = `UPDATE public.generation_jobs +SET status = $2::text, error = $3::jsonb, billing = $4::jsonb, + completed_at = COALESCE(completed_at, now()), locked_at = NULL, locked_by = NULL, updated_at = now() +WHERE id = $1::text RETURNING id` func (db *Database) ListJobs(ctx context.Context, filter jobs.ListFilter) ([]jobs.Job, error) { if err := db.requirePostgres("list generation jobs"); err != nil { @@ -75,11 +86,11 @@ func (db *Database) CreateJob(ctx context.Context, job jobs.Job) (jobs.Job, erro return jobs.Job{}, err } const columns = `id, owner_id, external_client_id, capability, provider, req_key, status, prompt, -input_asset_ids, input_urls, output_asset_ids, provider_task_id, request_payload, response_payload, +input_asset_ids, input_urls, output_asset_ids, provider_task_id, provider_dispatch_started_at, request_payload, response_payload, error, retry_of, idempotency_key, idempotency_fingerprint, priority, attempts, max_attempts, scheduled_at, locked_at, locked_by, started_at, completed_at, webhook_url, webhook_attempts, -webhook_last_status, usage_context, billing, created_at, updated_at` - placeholders := make([]string, 33) +webhook_last_status, usage_context, billing, dispatch_ready_at, finalized_at, created_at, updated_at` + placeholders := make([]string, 36) for index := range placeholders { placeholders[index] = fmt.Sprintf("$%d", index+1) } @@ -117,6 +128,9 @@ func (db *Database) UpdateJob(ctx context.Context, id string, patch jobs.Patch) } add("error", "::jsonb", raw) } + if patch.ClearError { + sets = append(sets, "error = NULL") + } if patch.Attempts != nil { add("attempts", "::integer", *patch.Attempts) } @@ -126,12 +140,27 @@ func (db *Database) UpdateJob(ctx context.Context, id string, patch jobs.Patch) if patch.CompletedAt != nil { add("completed_at", "::timestamptz", *patch.CompletedAt) } + if patch.DispatchReadyAt != nil { + add("dispatch_ready_at", "::timestamptz", *patch.DispatchReadyAt) + } + if patch.FinalizedAt != nil { + add("finalized_at", "::timestamptz", *patch.FinalizedAt) + } if patch.ProviderTaskID != nil { add("provider_task_id", "::text", *patch.ProviderTaskID) } + if patch.ProviderDispatchStartedAt != nil { + add("provider_dispatch_started_at", "::timestamptz", *patch.ProviderDispatchStartedAt) + } + if patch.SetResponsePayload { + add("response_payload", "::jsonb", optionalJSON(patch.ResponsePayload)) + } if patch.ClearProviderTaskID { sets = append(sets, "provider_task_id = NULL") } + if patch.ClearProviderDispatch { + sets = append(sets, "provider_dispatch_started_at = NULL") + } if patch.ClearLease { sets = append(sets, "locked_at = NULL", "locked_by = NULL") } @@ -145,17 +174,146 @@ func (db *Database) UpdateJob(ctx context.Context, id string, patch jobs.Patch) return db.mustFindJobByID(ctx, id) } sets = append(sets, "updated_at = now()") - query := `UPDATE public.generation_jobs SET ` + strings.Join(sets, ", ") + ` WHERE id = $1::text RETURNING ` + jobColumns + where := []string{"id = $1::text"} + if len(patch.ExpectedStatuses) != 0 { + values := make([]string, len(patch.ExpectedStatuses)) + for index, status := range patch.ExpectedStatuses { + values[index] = string(status) + } + args = append(args, values) + where = append(where, fmt.Sprintf("status = ANY($%d::text[])", len(args))) + } + if patch.ExpectedLockedBy != nil { + args = append(args, *patch.ExpectedLockedBy) + where = append(where, fmt.Sprintf("locked_by = $%d::text", len(args))) + } + query := `UPDATE public.generation_jobs SET ` + strings.Join(sets, ", ") + ` WHERE ` + strings.Join(where, " AND ") + ` RETURNING ` + jobColumns job, found, err := db.queryOneJob(ctx, query, args...) if err != nil { return jobs.Job{}, fmt.Errorf("update generation job: %w", err) } if !found { + if len(patch.ExpectedStatuses) != 0 || patch.ExpectedLockedBy != nil { + return jobs.Job{}, jobs.ErrTransitionConflict + } return jobs.Job{}, fmt.Errorf("generation job not found: %s", id) } return job, nil } +func (db *Database) DeleteJob(ctx context.Context, id string) error { + if err := db.requirePostgres("delete generation job"); err != nil { + return err + } + rows, err := db.querier.Query(ctx, `DELETE FROM public.generation_jobs WHERE id = $1::text RETURNING id`, id) + if err != nil { + return fmt.Errorf("delete generation job: %w", err) + } + defer rows.Close() + if !rows.Next() { + return fmt.Errorf("generation job not found: %s", id) + } + var deleted string + if err := rows.Scan(&deleted); err != nil { + return fmt.Errorf("scan deleted generation job: %w", err) + } + return nil +} + +// WriteBilling and WriteOutputAssetIDs implement the deliberately narrow +// orchestration state seam without widening jobs.Patch into a generic update +// bag. Both fail closed when the target disappeared during processing. +func (db *Database) WriteBilling(ctx context.Context, id string, value json.RawMessage) error { + if len(value) == 0 || !json.Valid(value) { + return fmt.Errorf("write generation billing: invalid JSON") + } + return db.writeJobState(ctx, "write generation billing", WriteJobBillingSQL, id, value) +} + +func (db *Database) ActivateCreation(ctx context.Context, id string, value json.RawMessage) error { + if len(value) == 0 || !json.Valid(value) { + return fmt.Errorf("activate generation creation: invalid JSON") + } + return db.writeJobState(ctx, "activate generation creation", ActivateJobCreationSQL, id, value) +} + +func (db *Database) WriteOutputAssetIDs(ctx context.Context, id string, values []string) error { + if values == nil { + values = []string{} + } + return db.writeJobState(ctx, "write generation output assets", WriteJobOutputAssetIDsSQL, id, values) +} + +func (db *Database) WriteBillingFenced(ctx context.Context, id string, value json.RawMessage, status jobs.Status, lockedBy string) error { + if len(value) == 0 || !json.Valid(value) || lockedBy == "" { + return fmt.Errorf("write generation billing: invalid fenced state") + } + return db.writeJobStateConflict(ctx, "write generation billing", WriteJobBillingFencedSQL, id, value, string(status), lockedBy) +} + +func (db *Database) WriteOutputAssetIDsFenced(ctx context.Context, id string, values []string, status jobs.Status, lockedBy string) error { + if lockedBy == "" { + return fmt.Errorf("write generation output assets: invalid fenced state") + } + if values == nil { + values = []string{} + } + return db.writeJobStateConflict(ctx, "write generation output assets", WriteJobOutputAssetIDsFencedSQL, id, values, string(status), lockedBy) +} + +// FailCreation commits the externally observable result of a failed charge in +// one statement. A job must never remain queued with a wallet-side failure. +func (db *Database) FailCreation(ctx context.Context, job jobs.Job) error { + if job.Status != jobs.StatusFailed || job.Error == nil || len(job.Billing) == 0 || !json.Valid(job.Billing) { + return fmt.Errorf("fail generation creation: invalid terminal state") + } + errorJSON, err := json.Marshal(job.Error) + if err != nil { + return fmt.Errorf("fail generation creation: encode error: %w", err) + } + return db.writeJobStateArgs(ctx, "fail generation creation", FailJobCreationSQL, job.ID, string(job.Status), json.RawMessage(errorJSON), job.Billing) +} + +func (db *Database) writeJobState(ctx context.Context, operation, query, id string, value any) error { + return db.writeJobStateArgs(ctx, operation, query, id, value) +} + +func (db *Database) writeJobStateArgs(ctx context.Context, operation, query, id string, values ...any) error { + if err := db.requirePostgres(operation); err != nil { + return err + } + args := make([]any, 0, len(values)+1) + args = append(args, id) + args = append(args, values...) + rows, err := db.querier.Query(ctx, query, args...) + if err != nil { + return fmt.Errorf("%s: %w", operation, err) + } + defer rows.Close() + if !rows.Next() { + if err := rows.Err(); err != nil { + return fmt.Errorf("%s: %w", operation, err) + } + return fmt.Errorf("%s: generation job not found", operation) + } + var returnedID string + if err := rows.Scan(&returnedID); err != nil { + return fmt.Errorf("%s: scan result: %w", operation, err) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("%s: %w", operation, err) + } + return nil +} + +func (db *Database) writeJobStateConflict(ctx context.Context, operation, query, id string, values ...any) error { + err := db.writeJobStateArgs(ctx, operation, query, id, values...) + if err != nil && strings.Contains(err.Error(), "generation job not found") { + return jobs.ErrTransitionConflict + } + return err +} + func (db *Database) ClaimJobs(ctx context.Context, workerID string, limit, lockTimeoutSeconds int) ([]jobs.Job, error) { if err := db.requirePostgres("claim generation jobs"); err != nil { return nil, err @@ -221,15 +379,15 @@ func scanJob(rows Rows) (jobs.Job, error) { var job jobs.Job var externalClientID, prompt, providerTaskID, retryOf, idempotencyKey, fingerprint sql.NullString var lockedBy, webhookURL sql.NullString - var lockedAt, startedAt, completedAt sql.NullTime + var lockedAt, startedAt, completedAt, dispatchReadyAt, finalizedAt, providerDispatchStartedAt sql.NullTime var requestPayload, responsePayload, errorPayload, webhookStatus, usageContext, billing []byte var status string err := rows.Scan( &job.ID, &job.OwnerID, &externalClientID, &job.Capability, &job.Provider, &job.ReqKey, &status, &prompt, - &job.InputAssetIDs, &job.InputURLs, &job.OutputAssetIDs, &providerTaskID, &requestPayload, &responsePayload, + &job.InputAssetIDs, &job.InputURLs, &job.OutputAssetIDs, &providerTaskID, &providerDispatchStartedAt, &requestPayload, &responsePayload, &errorPayload, &retryOf, &idempotencyKey, &fingerprint, &job.Priority, &job.Attempts, &job.MaxAttempts, &job.ScheduledAt, &lockedAt, &lockedBy, &startedAt, &completedAt, &webhookURL, &job.WebhookAttempts, - &webhookStatus, &usageContext, &billing, &job.CreatedAt, &job.UpdatedAt, + &webhookStatus, &usageContext, &billing, &dispatchReadyAt, &finalizedAt, &job.CreatedAt, &job.UpdatedAt, ) if err != nil { return jobs.Job{}, fmt.Errorf("scan generation job: %w", err) @@ -238,6 +396,7 @@ func scanJob(rows Rows) (jobs.Job, error) { job.ExternalClientID = nullString(externalClientID) job.Prompt = nullString(prompt) job.ProviderTaskID = nullString(providerTaskID) + job.ProviderDispatchStartedAt = nullTimePointer(providerDispatchStartedAt) job.RetryOf = nullString(retryOf) job.IdempotencyKey = nullString(idempotencyKey) job.IdempotencyFingerprint = nullString(fingerprint) @@ -246,6 +405,8 @@ func scanJob(rows Rows) (jobs.Job, error) { job.LockedAt = nullTimePointer(lockedAt) job.StartedAt = nullTimePointer(startedAt) job.CompletedAt = nullTimePointer(completedAt) + job.DispatchReadyAt = nullTimePointer(dispatchReadyAt) + job.FinalizedAt = nullTimePointer(finalizedAt) job.RequestPayload = normalizedJSON(requestPayload, `{}`) job.ResponsePayload = normalizedOptionalJSON(responsePayload) job.WebhookLastStatus = normalizedOptionalJSON(webhookStatus) @@ -276,11 +437,11 @@ func jobArguments(job jobs.Job) []any { return []any{ job.ID, job.OwnerID, optionalDatabaseText(job.ExternalClientID), job.Capability, job.Provider, job.ReqKey, string(job.Status), optionalDatabaseText(job.Prompt), job.InputAssetIDs, job.InputURLs, job.OutputAssetIDs, - optionalDatabaseText(job.ProviderTaskID), job.RequestPayload, optionalJSON(job.ResponsePayload), optionalJSON(errorPayload), + optionalDatabaseText(job.ProviderTaskID), job.ProviderDispatchStartedAt, job.RequestPayload, optionalJSON(job.ResponsePayload), optionalJSON(errorPayload), optionalDatabaseText(job.RetryOf), optionalDatabaseText(job.IdempotencyKey), optionalDatabaseText(job.IdempotencyFingerprint), job.Priority, job.Attempts, job.MaxAttempts, job.ScheduledAt, job.LockedAt, optionalDatabaseText(job.LockedBy), job.StartedAt, job.CompletedAt, optionalDatabaseText(job.WebhookURL), job.WebhookAttempts, - optionalJSON(job.WebhookLastStatus), optionalJSON(job.UsageContext), optionalJSON(job.Billing), job.CreatedAt, job.UpdatedAt, + optionalJSON(job.WebhookLastStatus), optionalJSON(job.UsageContext), optionalJSON(job.Billing), job.DispatchReadyAt, job.FinalizedAt, job.CreatedAt, job.UpdatedAt, } } diff --git a/backend/internal/postgres/jobs_test.go b/backend/internal/postgres/jobs_test.go index 44134c9..7d794ba 100644 --- a/backend/internal/postgres/jobs_test.go +++ b/backend/internal/postgres/jobs_test.go @@ -78,6 +78,135 @@ func TestJobsAdapterUpdateClearsLeaseAndProviderTask(t *testing.T) { } } +func TestJobsAdapterUpdateClearsStaleProviderError(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + querier := &jobQuerier{rows: &jobRows{rows: [][]any{jobRow(now)}}} + database := NewDatabase(Config{Backend: BackendPostgres}, querier) + status := jobs.StatusSucceeded + + if _, err := database.UpdateJob(context.Background(), "job-1", jobs.Patch{Status: &status, ClearError: true}); err != nil { + t.Fatal(err) + } + if !strings.Contains(querier.query, "error = NULL") { + t.Fatalf("update query does not clear stale error: %s", querier.query) + } +} + +func TestJobsAdapterFencesWorkerUpdateByExpectedStateAndLeaseOwner(t *testing.T) { + now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC) + querier := &jobQuerier{rows: &jobRows{rows: [][]any{jobRow(now)}}} + database := NewDatabase(Config{Backend: BackendPostgres}, querier) + status := jobs.StatusSucceeded + worker := "worker-1" + _, err := database.UpdateJob(context.Background(), "job-1", jobs.Patch{Status: &status, ExpectedStatuses: []jobs.Status{jobs.StatusRunning}, ExpectedLockedBy: &worker}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(querier.query, "status = ANY(") || !strings.Contains(querier.query, "locked_by =") { + t.Fatalf("fenced update query = %s", querier.query) + } + + querier.rows = &jobRows{} + _, err = database.UpdateJob(context.Background(), "job-1", jobs.Patch{Status: &status, ExpectedStatuses: []jobs.Status{jobs.StatusRunning}, ExpectedLockedBy: &worker}) + if !errors.Is(err, jobs.ErrTransitionConflict) { + t.Fatalf("lost lease error = %v", err) + } +} + +func TestJobsAdapterActivatesChargedCreationAtomically(t *testing.T) { + querier := &jobQuerier{rows: &jobRows{rows: [][]any{{"job-1"}}}} + database := NewDatabase(Config{Backend: BackendPostgres}, querier) + if err := database.ActivateCreation(context.Background(), "job-1", json.RawMessage(`{"status":"charged"}`)); err != nil { + t.Fatal(err) + } + if querier.query != ActivateJobCreationSQL || !strings.Contains(querier.query, "dispatch_ready_at") { + t.Fatalf("activation query = %s", querier.query) + } +} + +func TestJobsAdapterActivationReplayIsIdempotentForSameBilling(t *testing.T) { + querier := &jobQuerier{rows: &jobRows{rows: [][]any{{"job-1"}}}} + database := NewDatabase(Config{Backend: BackendPostgres}, querier) + billingState := json.RawMessage(`{"status":"charged","ledgerEntryId":"ledger-1"}`) + if err := database.ActivateCreation(context.Background(), "job-1", billingState); err != nil { + t.Fatal(err) + } + querier.rows = &jobRows{rows: [][]any{{"job-1"}}} + if err := database.ActivateCreation(context.Background(), "job-1", billingState); err != nil { + t.Fatalf("replay: %v", err) + } + if !strings.Contains(querier.query, "dispatch_ready_at IS NULL OR billing = $2::jsonb") { + t.Fatalf("activation is not idempotent for matching billing: %s", querier.query) + } +} + +func TestJobsAdapterWritesBillingAndOutputAssetsThroughNarrowStateSeam(t *testing.T) { + querier := &jobQuerier{rows: &jobRows{rows: [][]any{{"job-1"}}}} + database := NewDatabase(Config{Backend: BackendPostgres}, querier) + + if err := database.WriteBilling(context.Background(), "job-1", json.RawMessage(`{"status":"refunded"}`)); err != nil { + t.Fatal(err) + } + if querier.query != WriteJobBillingSQL || !reflect.DeepEqual(querier.args, []any{"job-1", json.RawMessage(`{"status":"refunded"}`)}) { + t.Fatalf("billing query=%q args=%#v", querier.query, querier.args) + } + + querier.rows = &jobRows{rows: [][]any{{"job-1"}}} + if err := database.WriteOutputAssetIDs(context.Background(), "job-1", []string{"asset-1", "asset-2"}); err != nil { + t.Fatal(err) + } + if querier.query != WriteJobOutputAssetIDsSQL || !reflect.DeepEqual(querier.args, []any{"job-1", []string{"asset-1", "asset-2"}}) { + t.Fatalf("outputs query=%q args=%#v", querier.query, querier.args) + } +} + +func TestJobsAdapterFencesOrchestrationStateWrites(t *testing.T) { + querier := &jobQuerier{rows: &jobRows{rows: [][]any{{"job-1"}}}} + database := NewDatabase(Config{Backend: BackendPostgres}, querier) + if err := database.WriteBillingFenced(context.Background(), "job-1", json.RawMessage(`{"status":"settled"}`), jobs.StatusSucceeded, "worker-1"); err != nil { + t.Fatal(err) + } + if querier.query != WriteJobBillingFencedSQL || !reflect.DeepEqual(querier.args, []any{"job-1", json.RawMessage(`{"status":"settled"}`), "succeeded", "worker-1"}) { + t.Fatalf("query=%q args=%#v", querier.query, querier.args) + } + querier.rows = &jobRows{} + if err := database.WriteOutputAssetIDsFenced(context.Background(), "job-1", []string{"asset"}, jobs.StatusSucceeded, "worker-1"); !errors.Is(err, jobs.ErrTransitionConflict) { + t.Fatalf("lost lease error=%v", err) + } +} + +func TestJobsAdapterPersistsFailedCreationAtomically(t *testing.T) { + querier := &jobQuerier{rows: &jobRows{rows: [][]any{{"job-1"}}}} + database := NewDatabase(Config{Backend: BackendPostgres}, querier) + job := jobs.Job{ + ID: "job-1", Status: jobs.StatusFailed, + Error: &jobs.JobError{Message: "generation charge failed"}, + Billing: json.RawMessage(`{"status":"not_charged"}`), + } + + if err := database.FailCreation(context.Background(), job); err != nil { + t.Fatal(err) + } + if querier.query != FailJobCreationSQL { + t.Fatalf("query=%q", querier.query) + } + if len(querier.args) != 4 || querier.args[0] != "job-1" || querier.args[1] != string(jobs.StatusFailed) || + !reflect.DeepEqual(querier.args[2], json.RawMessage(`{"message":"generation charge failed"}`)) || + !reflect.DeepEqual(querier.args[3], job.Billing) { + t.Fatalf("args=%#v", querier.args) + } +} + +func TestJobsAdapterStateWritesFailWhenJobIsMissing(t *testing.T) { + database := NewDatabase(Config{Backend: BackendPostgres}, &jobQuerier{rows: &jobRows{}}) + if err := database.WriteBilling(context.Background(), "missing", json.RawMessage(`{}`)); err == nil { + t.Fatal("WriteBilling error=nil") + } + if err := database.WriteOutputAssetIDs(context.Background(), "missing", []string{}); err == nil { + t.Fatal("WriteOutputAssetIDs error=nil") + } +} + func TestJobsAdapterFailsClosedWithoutPostgres(t *testing.T) { database := NewDatabase(Config{Backend: BackendLocal}, nil) if _, err := database.ClaimJobs(context.Background(), "worker", 1, 300); err == nil { @@ -91,9 +220,9 @@ func TestJobsAdapterFailsClosedWithoutPostgres(t *testing.T) { func jobRow(now time.Time) []any { return []any{ "job-1", "api:client", "client", "image.generate", "mock", "fixture", "queued", "prompt", - []string{}, []string{}, []string{}, nil, []byte(`{"input":true}`), nil, nil, nil, + []string{}, []string{}, []string{}, nil, nil, []byte(`{"input":true}`), nil, nil, nil, "idem-1", "fingerprint", 5, 0, 3, now, nil, nil, nil, nil, "https://hooks.example.test", 0, - nil, []byte(`{"source":"api","accountId":"client","displayName":"client"}`), nil, now, now, + nil, []byte(`{"source":"api","accountId":"client","displayName":"client"}`), nil, now, nil, now, now, } } @@ -135,8 +264,12 @@ func (rows *jobRows) Scan(dest ...any) error { } case *int: *target = value.(int) + case *int64: + *target = value.(int64) case *time.Time: - *target = value.(time.Time) + if value != nil { + *target = value.(time.Time) + } case *sql.NullString: if value != nil { *target = sql.NullString{String: value.(string), Valid: true} @@ -151,6 +284,8 @@ func (rows *jobRows) Scan(dest ...any) error { if value != nil { *target = append([]byte(nil), value.([]byte)...) } + case *any: + *target = value default: return errors.New("unsupported scan target") } diff --git a/backend/internal/postgres/password_change.go b/backend/internal/postgres/password_change.go new file mode 100644 index 0000000..d870fb8 --- /dev/null +++ b/backend/internal/postgres/password_change.go @@ -0,0 +1,126 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration" + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity" +) + +const SelectPasswordChangeAccountSQL = `SELECT + id, phone, display_name, role, organization_id, status, + password_hash, password_salt, session_version +FROM public.platform_users +WHERE id = $1::text +FOR UPDATE` + +const UpdatePasswordChangeAccountSQL = `UPDATE public.platform_users +SET password_hash = $2, + password_salt = $3, + session_version = session_version + 1, + updated_at = $4 +WHERE id = $1::text +RETURNING id, phone, display_name, role, organization_id, status, session_version` + +// ChangeOwnPassword serializes password changes on the account row so two +// concurrent requests cannot both verify the same previous credential. +func (db *Database) ChangeOwnPassword(ctx context.Context, accountID, currentPassword, nextPassword string, now time.Time) (identity.AuthorizationSnapshot, error) { + if db.config.Backend != BackendPostgres || db.transactions == nil { + return identity.AuthorizationSnapshot{}, fmt.Errorf("PostgreSQL is unavailable when ZHINIAN_DATA_BACKEND=%s", db.config.Backend) + } + tx, err := db.transactions.Begin(ctx) + if err != nil { + return identity.AuthorizationSnapshot{}, fmt.Errorf("begin password change transaction: %w", err) + } + finished := false + defer func() { + if !finished { + _ = tx.Rollback(ctx) + } + }() + + account, hash, salt, found, err := loadPasswordChangeAccount(ctx, tx, accountID) + if err != nil { + return identity.AuthorizationSnapshot{}, err + } + if !found || account.Status != "active" { + return identity.AuthorizationSnapshot{}, &identity.PasswordChangeError{Reason: identity.PasswordChangeNotFound} + } + valid, err := verifyNodeScryptPassword(currentPassword, hash, salt) + if err != nil { + return identity.AuthorizationSnapshot{}, fmt.Errorf("verify current password: %w", err) + } + if !valid { + return identity.AuthorizationSnapshot{}, &identity.PasswordChangeError{Reason: identity.PasswordChangeCurrentIncorrect} + } + password, err := administration.HashPassword(nextPassword) + if err != nil { + return identity.AuthorizationSnapshot{}, fmt.Errorf("hash new password: %w", err) + } + updated, err := updatePasswordChangeAccount(ctx, tx, accountID, password, now) + if err != nil { + return identity.AuthorizationSnapshot{}, err + } + + snapshot := identity.AuthorizationSnapshot{Account: updated} + if updated.OrganizationID != "" { + organization, found, err := loadPasswordLoginOrganization(ctx, tx, updated.OrganizationID) + if err != nil { + return identity.AuthorizationSnapshot{}, err + } + if found { + snapshot.Organization = &organization + } + } + if err := tx.Commit(ctx); err != nil { + return identity.AuthorizationSnapshot{}, fmt.Errorf("commit password change transaction: %w", err) + } + finished = true + return snapshot, nil +} + +func loadPasswordChangeAccount(ctx context.Context, tx Transaction, id string) (identity.AccountSnapshot, string, string, bool, error) { + rows, err := tx.Query(ctx, SelectPasswordChangeAccountSQL, id) + if err != nil { + return identity.AccountSnapshot{}, "", "", false, fmt.Errorf("query password change account: %w", err) + } + defer rows.Close() + if !rows.Next() { + return identity.AccountSnapshot{}, "", "", false, rows.Err() + } + var account identity.AccountSnapshot + var organizationID sql.NullString + var hash, salt string + if err := rows.Scan(&account.ID, &account.Phone, &account.DisplayName, &account.Role, &organizationID, &account.Status, &hash, &salt, &account.SessionVersion); err != nil { + return identity.AccountSnapshot{}, "", "", false, fmt.Errorf("scan password change account: %w", err) + } + if organizationID.Valid { + account.OrganizationID = organizationID.String + } + return account, hash, salt, true, rows.Err() +} + +func updatePasswordChangeAccount(ctx context.Context, tx Transaction, id string, password administration.PasswordHash, now time.Time) (identity.AccountSnapshot, error) { + rows, err := tx.Query(ctx, UpdatePasswordChangeAccountSQL, id, password.Hash, password.Salt, now) + if err != nil { + return identity.AccountSnapshot{}, fmt.Errorf("update password change account: %w", err) + } + defer rows.Close() + if !rows.Next() { + return identity.AccountSnapshot{}, &identity.PasswordChangeError{Reason: identity.PasswordChangeNotFound} + } + var account identity.AccountSnapshot + var organizationID sql.NullString + if err := rows.Scan(&account.ID, &account.Phone, &account.DisplayName, &account.Role, &organizationID, &account.Status, &account.SessionVersion); err != nil { + return identity.AccountSnapshot{}, fmt.Errorf("scan updated password change account: %w", err) + } + if organizationID.Valid { + account.OrganizationID = organizationID.String + } + return account, rows.Err() +} + +var _ identity.PasswordChanger = (*Database)(nil) diff --git a/backend/internal/postgres/password_change_test.go b/backend/internal/postgres/password_change_test.go new file mode 100644 index 0000000..b4456a8 --- /dev/null +++ b/backend/internal/postgres/password_change_test.go @@ -0,0 +1,48 @@ +package postgres + +import ( + "context" + "errors" + "reflect" + "testing" + "time" +) + +func TestChangeOwnPasswordLocksVerifiesUpdatesAndRefreshesSnapshot(t *testing.T) { + now := time.Date(2026, 8, 13, 12, 0, 0, 0, time.UTC) + hash := nodeCompatibleHash(t, "current-password", "salt") + tx := &loginTransaction{queries: []loginQueryResult{ + {rows: loginRows([]any{"user-1", "13800138000", "User", "user", "org-1", "active", hash, "salt", 4})}, + {rows: loginRows([]any{"user-1", "13800138000", "User", "user", "org-1", "active", 5})}, + {rows: loginRows([]any{"org-1", "Acme", "active"})}, + }} + got, err := loginDatabase(tx).ChangeOwnPassword(context.Background(), "user-1", "current-password", "next-password", now) + if err != nil { + t.Fatal(err) + } + if got.Account.SessionVersion != 5 || got.Organization == nil || got.Organization.Name != "Acme" || tx.commits != 1 { + t.Fatalf("snapshot=%#v commits=%d", got, tx.commits) + } + if tx.queriesSeen[0].sql != SelectPasswordChangeAccountSQL || tx.queriesSeen[1].sql != UpdatePasswordChangeAccountSQL || tx.queriesSeen[2].sql != SelectPasswordLoginOrganizationSQL { + t.Fatalf("queries=%#v", tx.queriesSeen) + } + if !reflect.DeepEqual(tx.queriesSeen[0].args, []any{"user-1"}) { + t.Fatalf("args=%#v", tx.queriesSeen[0].args) + } +} + +func TestChangeOwnPasswordRejectsCurrentPasswordAndRollsBack(t *testing.T) { + tx := &loginTransaction{queries: []loginQueryResult{{rows: loginRows([]any{"u", "p", "N", "super_admin", nil, "active", nodeCompatibleHash(t, "right", "salt"), "salt", 1})}}} + _, err := loginDatabase(tx).ChangeOwnPassword(context.Background(), "u", "wrong", "next-password", time.Now()) + if err == nil || tx.commits != 0 || tx.rollbacks != 1 || len(tx.queriesSeen) != 1 { + t.Fatalf("err=%v tx=%#v", err, tx) + } +} + +func TestChangeOwnPasswordRollsBackInfrastructureFailure(t *testing.T) { + tx := &loginTransaction{queries: []loginQueryResult{{err: errors.New("boom")}}} + _, err := loginDatabase(tx).ChangeOwnPassword(context.Background(), "u", "old-password", "next-password", time.Now()) + if err == nil || tx.rollbacks != 1 { + t.Fatalf("err=%v rollbacks=%d", err, tx.rollbacks) + } +} diff --git a/backend/internal/postgres/templates.go b/backend/internal/postgres/templates.go new file mode 100644 index 0000000..fec4ab4 --- /dev/null +++ b/backend/internal/postgres/templates.go @@ -0,0 +1,136 @@ +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates" +) + +const templateColumns = `id, owner_id, name, description, prompt, preview_image_url, settings, sort_order, created_at, updated_at` + +const ListImageTemplatesSQL = `SELECT ` + templateColumns + ` FROM public.image_templates WHERE owner_id = $1::text ORDER BY sort_order ASC, updated_at DESC` +const CreateImageTemplateSQL = `INSERT INTO public.image_templates (id, owner_id, name, description, prompt, preview_image_url, settings, sort_order, created_at, updated_at) VALUES ($1::text,$2::text,$3::text,$4::text,$5::text,$6::text,$7::jsonb,$8::integer,$9::timestamptz,$10::timestamptz) RETURNING ` + templateColumns +const UpdateImageTemplateSQL = `UPDATE public.image_templates SET name=COALESCE($3::text,name), description=CASE WHEN $4::boolean THEN $5::text ELSE description END, prompt=COALESCE($6::text,prompt), preview_image_url=CASE WHEN $7::boolean THEN $8::text ELSE preview_image_url END, settings=COALESCE($9::jsonb,settings), sort_order=COALESCE($10::integer,sort_order), updated_at=$11::timestamptz WHERE owner_id=$1::text AND id=$2::text RETURNING ` + templateColumns +const DeleteImageTemplateSQL = `DELETE FROM public.image_templates WHERE owner_id=$1::text AND id=$2::text RETURNING ` + templateColumns + +func (db *Database) ListTemplates(ctx context.Context, owner string) ([]templates.Template, error) { + if err := db.available(); err != nil { + return nil, err + } + rows, err := db.querier.Query(ctx, ListImageTemplatesSQL, owner) + if err != nil { + return nil, fmt.Errorf("list image templates: %w", err) + } + defer rows.Close() + items := []templates.Template{} + for rows.Next() { + item, current := scanTemplate(rows) + if current != nil { + return nil, current + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read image templates: %w", err) + } + return items, nil +} + +func (db *Database) CreateTemplate(ctx context.Context, item templates.Template) (templates.Template, error) { + settings, err := json.Marshal(item.Settings) + if err != nil { + return templates.Template{}, fmt.Errorf("encode template settings: %w", err) + } + return db.requiredTemplate(ctx, CreateImageTemplateSQL, item.ID, item.OwnerID, item.Name, nullableText(item.Description), item.Prompt, nullableText(item.PreviewImageURL), settings, item.SortOrder, item.CreatedAt, item.UpdatedAt) +} + +func (db *Database) UpdateTemplate(ctx context.Context, owner, id string, patch templates.Patch, now time.Time) (templates.Template, bool, error) { + var settings any + if patch.Settings != nil { + encoded, err := json.Marshal(*patch.Settings) + if err != nil { + return templates.Template{}, false, fmt.Errorf("encode template settings: %w", err) + } + settings = encoded + } + args := []any{owner, id, patch.Name, patch.Description != nil, pointerText(patch.Description), patch.Prompt, patch.PreviewImageURL != nil, pointerText(patch.PreviewImageURL), settings, patch.SortOrder, now} + return db.oneTemplate(ctx, UpdateImageTemplateSQL, args...) +} + +func (db *Database) DeleteTemplate(ctx context.Context, owner, id string) (templates.Template, bool, error) { + return db.oneTemplate(ctx, DeleteImageTemplateSQL, owner, id) +} + +func (db *Database) requiredTemplate(ctx context.Context, query string, args ...any) (templates.Template, error) { + item, found, err := db.oneTemplate(ctx, query, args...) + if err != nil { + return templates.Template{}, err + } + if !found { + return templates.Template{}, fmt.Errorf("image template write returned no row") + } + return item, nil +} +func (db *Database) oneTemplate(ctx context.Context, query string, args ...any) (templates.Template, bool, error) { + if err := db.available(); err != nil { + return templates.Template{}, false, err + } + rows, err := db.querier.Query(ctx, query, args...) + if err != nil { + return templates.Template{}, false, fmt.Errorf("query image template: %w", err) + } + defer rows.Close() + if !rows.Next() { + if err := rows.Err(); err != nil { + return templates.Template{}, false, fmt.Errorf("read image template: %w", err) + } + return templates.Template{}, false, nil + } + item, err := scanTemplate(rows) + if err != nil { + return templates.Template{}, false, err + } + if err := rows.Err(); err != nil { + return templates.Template{}, false, fmt.Errorf("read image template: %w", err) + } + return item, true, nil +} +func scanTemplate(rows Rows) (templates.Template, error) { + var item templates.Template + var description, preview sql.NullString + var settings []byte + if err := rows.Scan(&item.ID, &item.OwnerID, &item.Name, &description, &item.Prompt, &preview, &settings, &item.SortOrder, &item.CreatedAt, &item.UpdatedAt); err != nil { + return templates.Template{}, fmt.Errorf("scan image template: %w", err) + } + if description.Valid { + item.Description = description.String + } + if preview.Valid { + item.PreviewImageURL = preview.String + } + if len(settings) == 0 { + settings = []byte(`{}`) + } + if err := json.Unmarshal(settings, &item.Settings); err != nil { + return templates.Template{}, fmt.Errorf("decode template settings: %w", err) + } + return item, nil +} +func nullableText(value string) any { + if value == "" { + return nil + } + return value +} +func pointerText(value *string) any { + if value == nil || *value == "" { + return nil + } + return *value +} + +var _ templates.Catalog = (*Database)(nil) diff --git a/backend/internal/postgres/templates_test.go b/backend/internal/postgres/templates_test.go new file mode 100644 index 0000000..f20f347 --- /dev/null +++ b/backend/internal/postgres/templates_test.go @@ -0,0 +1,86 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + "git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates" +) + +func TestImageTemplateCatalogUsesOwnerScopedExplicitSQL(t *testing.T) { + now := time.Date(2026, 8, 13, 0, 0, 0, 0, time.UTC) + querier := &templateQuerier{rows: &templateRows{rows: [][]any{{"t1", "owner", "Name", nil, "Prompt", nil, []byte(`{"engine":"jimeng"}`), 2, now, now}}}} + db := NewDatabase(Config{Backend: BackendPostgres}, querier) + items, err := db.ListTemplates(context.Background(), "owner") + if err != nil || len(items) != 1 || items[0].Settings.Engine != "jimeng" { + t.Fatalf("items=%#v err=%v", items, err) + } + if querier.query != ListImageTemplatesSQL || len(querier.args) != 1 || querier.args[0] != "owner" { + t.Fatalf("query=%q args=%#v", querier.query, querier.args) + } +} + +func TestImageTemplateUpdatePreservesOmittedNullableFields(t *testing.T) { + now := time.Now().UTC() + querier := &templateQuerier{rows: &templateRows{rows: [][]any{{"t1", "owner", "Name", "desc", "Prompt", "/p", []byte(`{}`), 0, now, now}}}} + db := NewDatabase(Config{Backend: BackendPostgres}, querier) + name := "Name" + item, found, err := db.UpdateTemplate(context.Background(), "owner", "t1", templates.Patch{Name: &name}, now) + if err != nil || !found || item.ID != "t1" { + t.Fatalf("item=%#v found=%v err=%v", item, found, err) + } + if querier.query != UpdateImageTemplateSQL || querier.args[3] != false || querier.args[6] != false { + t.Fatalf("unexpected args %#v", querier.args) + } +} + +type templateQuerier struct { + rows Rows + query string + args []any +} + +func (querier *templateQuerier) Query(_ context.Context, query string, args ...any) (Rows, error) { + querier.query = query + querier.args = args + return querier.rows, nil +} + +type templateRows struct { + rows [][]any + index int +} + +func (*templateRows) Close() {} +func (*templateRows) Err() error { return nil } +func (rows *templateRows) Next() bool { return rows.index < len(rows.rows) } +func (rows *templateRows) Scan(dest ...any) error { + row := rows.rows[rows.index] + rows.index++ + if len(dest) != len(row) { + return fmt.Errorf("scan arity") + } + for index, value := range row { + switch target := dest[index].(type) { + case *string: + if value != nil { + *target = value.(string) + } + case *sql.NullString: + if value != nil { + target.String = value.(string) + target.Valid = true + } + case *[]byte: + *target = value.([]byte) + case *int: + *target = value.(int) + case *time.Time: + *target = value.(time.Time) + } + } + return nil +} diff --git a/backend/internal/postgres/usage.go b/backend/internal/postgres/usage.go index 33a2a1d..202391d 100644 --- a/backend/internal/postgres/usage.go +++ b/backend/internal/postgres/usage.go @@ -85,5 +85,8 @@ func (repository UsageRepository) Insert(event usage.Event) (usage.Event, bool, func (repository UsageRepository) List(filters usage.Filters) ([]usage.Event, error) { return repository.database.ListUsageEvents(context.Background(), filters) } +func (repository UsageRepository) ListContext(ctx context.Context, filters usage.Filters) ([]usage.Event, error) { + return repository.database.ListUsageEvents(ctx, filters) +} var _ usage.Repository = UsageRepository{} diff --git a/backend/internal/prompt/assembler.go b/backend/internal/prompt/assembler.go new file mode 100644 index 0000000..aba98a6 --- /dev/null +++ b/backend/internal/prompt/assembler.go @@ -0,0 +1,245 @@ +// Package prompt implements the deterministic prompt-assembly contract used by +// browser and public generation requests. +package prompt + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +type Scene struct { + ID string `json:"id"` + Title string `json:"title"` + Visual string `json:"visual"` + Camera string `json:"camera,omitempty"` + HostLine string `json:"hostLine,omitempty"` + Caption string `json:"caption,omitempty"` + MaterialLabel string `json:"materialLabel,omitempty"` +} +type Material struct { + ID string `json:"id,omitempty"` + URL string `json:"url"` + Type string `json:"type"` + Role string `json:"role,omitempty"` + Label string `json:"label,omitempty"` + Name string `json:"name,omitempty"` +} +type Input struct { + Mode string `json:"mode"` + ProjectName string `json:"projectName,omitempty"` + Audience string `json:"audience,omitempty"` + Offer string `json:"offer,omitempty"` + BrandLine string `json:"brandLine,omitempty"` + ManualPrompt string `json:"manualPrompt,omitempty"` + Storyboard []Scene `json:"storyboard,omitempty"` + Materials []Material `json:"materials,omitempty"` + ImageGoal string `json:"imageGoal,omitempty"` + AspectRatio string `json:"aspectRatio,omitempty"` +} +type Requirements struct { + Image int `json:"image"` + Video int `json:"video"` + Audio int `json:"audio"` +} +type Result struct { + Prompt string `json:"prompt"` + Scenes []Scene `json:"scenes"` + Materials []Material `json:"materials"` + Warnings []string `json:"warnings"` + Blocked bool `json:"blocked"` + Requirements Requirements `json:"requirements"` +} + +var DefaultScenes = []Scene{ + {ID: "scene-1", Title: "开场画面", Visual: "用上传素材建立项目的第一印象,主体清晰,氛围干净", Camera: "中景或推进镜头", Caption: "项目亮相"}, + {ID: "scene-2", Title: "场景氛围", Visual: "展示空间、环境或使用场景,让观众理解项目所处的真实语境", Camera: "横移或环绕", Caption: "场景氛围"}, + {ID: "scene-3", Title: "核心内容", Visual: "突出核心产品、服务、活动、空间或人物,呈现最重要的信息", Camera: "主体特写", Caption: "核心内容"}, + {ID: "scene-4", Title: "细节补充", Visual: "补充质感、服务、流程、环境或亮点细节,增强可信度", Camera: "细节切镜", Caption: "细节补充"}, + {ID: "scene-5", Title: "收尾画面", Visual: "用项目名称、品牌信息或完整画面收束,形成清楚的结束印象", Camera: "定格或拉远", Caption: "项目记忆点"}, +} + +func Assemble(input Input) Result { + scenes := input.Storyboard + if len(scenes) == 0 { + scenes = append([]Scene(nil), DefaultScenes...) + } + materials := NormalizeMaterials(input.Materials) + text := strings.TrimSpace(input.ManualPrompt) + if text == "" { + if input.Mode == "image" { + text = assembleImage(input, scenes) + } else { + text = assembleVideo(input, scenes) + } + } + requirements := ExtractRequirements(text) + warnings := []string{} + available := Requirements{} + for _, material := range materials { + switch material.Type { + case "video": + available.Video++ + case "audio": + available.Audio++ + default: + available.Image++ + } + } + if requirements.Image > available.Image { + warnings = append(warnings, fmt.Sprintf("提示词引用到 @图片%d,当前只绑定了 %d 张图片。", requirements.Image, available.Image)) + } + if requirements.Video > available.Video { + warnings = append(warnings, fmt.Sprintf("提示词引用到 @视频%d,当前只绑定了 %d 个视频。", requirements.Video, available.Video)) + } + if requirements.Audio > available.Audio { + warnings = append(warnings, fmt.Sprintf("提示词引用到 @音频%d,当前只绑定了 %d 个音频。", requirements.Audio, available.Audio)) + } + return Result{Prompt: text, Scenes: scenes, Materials: materials, Warnings: warnings, Blocked: false, Requirements: requirements} +} + +func NormalizeMaterials(input []Material) []Material { + counters := map[string]int{"image": 0, "video": 0, "audio": 0} + out := make([]Material, 0, len(input)) + for _, material := range input { + if strings.TrimSpace(material.URL) == "" { + continue + } + if material.Type != "image" && material.Type != "video" && material.Type != "audio" { + material.Type = inferType(material.URL) + } + if material.Label == "" { + counters[material.Type]++ + material.Label = label(material.Type, counters[material.Type]) + } + out = append(out, material) + } + sort.SliceStable(out, func(i, j int) bool { return labelWeight(out[i].Label) < labelWeight(out[j].Label) }) + return out +} + +var placeholderPattern = regexp.MustCompile(`@(参考视频|图片|图|视频|音频)([0-9]+)`) + +func ExtractRequirements(text string) Requirements { + result := Requirements{} + for _, match := range placeholderPattern.FindAllStringSubmatch(text, -1) { + var index int + _, _ = fmt.Sscanf(match[2], "%d", &index) + if index < 1 { + continue + } + switch match[1] { + case "视频", "参考视频": + if index > result.Video { + result.Video = index + } + case "音频": + if index > result.Audio { + result.Audio = index + } + default: + if index > result.Image { + result.Image = index + } + } + } + return result +} + +func assembleImage(input Input, scenes []Scene) string { + project := fallback(input.ProjectName, "当前项目") + goal := fallback(input.ImageGoal, "生成可用于营销传播的主视觉图片") + ratio := fallback(input.AspectRatio, "1:1") + lines := []string{} + for index, scene := range scenes { + if index >= 4 { + break + } + material := scene.MaterialLabel + if material == "" { + material = fmt.Sprintf("@图片%d", index+1) + } + suffix := "" + if scene.Caption != "" { + suffix = ";文字元素=" + scene.Caption + } + lines = append(lines, fmt.Sprintf("%d. %s:参考素材=%s;画面要点=%s%s", index+1, scene.Title, material, scene.Visual, suffix)) + } + return fmt.Sprintf("营销图片生成。\n项目名称:%s\n目标:%s\n目标人群:%s\n表达重点:%s\n补充说明:%s\n整体风格真实、有设计感,适合品牌和社媒传播。\n素材引用:\n%s\n生成要求:\n- 严格参考提示词中的@图片素材,保持主体和关键信息一致。\n- 可以使用视频素材作为节奏、镜头或氛围参考,但最终输出单张图片。\n- 图片比例:%s。\n- 文字内容少而准确,避免错别字和无关标语。\n- 不额外添加与项目无关的信息。", project, fallback(input.ImageGoal, goal), fallback(input.Audience, "泛营销受众"), fallback(input.Offer, "突出产品、服务或活动核心卖点"), fallback(input.BrandLine, "保持干净、可信、可发布的视觉质感"), strings.Join(lines, "\n"), ratio) +} +func assembleVideo(input Input, scenes []Scene) string { + info := []string{"项目名称:" + fallback(input.ProjectName, "当前项目")} + if strings.TrimSpace(input.Audience) != "" { + info = append(info, "目标人群:"+strings.TrimSpace(input.Audience)) + } + if strings.TrimSpace(input.Offer) != "" { + info = append(info, "表达重点:"+strings.TrimSpace(input.Offer)) + } + if strings.TrimSpace(input.BrandLine) != "" { + info = append(info, "补充说明:"+strings.TrimSpace(input.BrandLine)) + } + lines := []string{} + for index, scene := range scenes { + material := scene.MaterialLabel + if material == "" { + material = fmt.Sprintf("@图片%d", index+1) + } + suffix := "" + if scene.Camera != "" { + suffix += ";镜头=" + scene.Camera + } + if scene.HostLine != "" { + suffix += ";口播=" + scene.HostLine + } + if scene.Caption != "" { + suffix += ";字幕=" + scene.Caption + } + lines = append(lines, fmt.Sprintf("%d. %s:素材参考=%s;内容方向=%s%s", index+1, scene.Title, material, scene.Visual, suffix)) + } + return fmt.Sprintf("通用营销宣传视频。\n参考风格:真实自然的营销宣传片,画面干净、节奏清楚、转场自然。\n%s\n内容结构:\n%s\n生成要求:\n- 以项目名称和@素材为准,不套用示例中的具体地点、人物、文案或品牌。\n- 图片素材用于控制主体、场景、商品和分镜;视频素材用于控制节奏、转场、运镜或参考风格。\n- 画面真实干净,主体清晰,转场自然,整体观感统一。\n- 如生成字幕,只保留简短标题或重点信息,避免大段文字。\n- 不额外添加与项目无关的信息。", strings.Join(info, "\n"), strings.Join(lines, "\n")) +} +func inferType(raw string) string { + lower := strings.ToLower(strings.Split(raw, "?")[0]) + for _, suffix := range []string{".mp4", ".mov", ".webm"} { + if strings.HasSuffix(lower, suffix) { + return "video" + } + } + for _, suffix := range []string{".mp3", ".wav", ".m4a", ".aac", ".flac"} { + if strings.HasSuffix(lower, suffix) { + return "audio" + } + } + return "image" +} +func label(kind string, index int) string { + if kind == "video" { + return fmt.Sprintf("@视频%d", index) + } + if kind == "audio" { + return fmt.Sprintf("@音频%d", index) + } + return fmt.Sprintf("@图片%d", index) +} +func labelWeight(value string) int { + matches := regexp.MustCompile(`^@(图片|图|视频|音频)([0-9]+)$`).FindStringSubmatch(value) + if len(matches) == 0 { + return 999 + } + var index int + _, _ = fmt.Sscanf(matches[2], "%d", &index) + base := 0 + if matches[1] == "视频" { + base = 100 + } else if matches[1] == "音频" { + base = 200 + } + return base + index +} +func fallback(value, fallback string) string { + if strings.TrimSpace(value) == "" { + return fallback + } + return strings.TrimSpace(value) +} diff --git a/backend/internal/prompt/assembler_test.go b/backend/internal/prompt/assembler_test.go new file mode 100644 index 0000000..e2f386d --- /dev/null +++ b/backend/internal/prompt/assembler_test.go @@ -0,0 +1,36 @@ +package prompt + +import "testing" + +func TestAssembleManualPromptNormalizesMaterialsAndWarnings(t *testing.T) { + result := Assemble(Input{Mode: "image", ManualPrompt: " use @图片2 and @视频1 ", Materials: []Material{ + {URL: "/b.mp4", Type: "video"}, {URL: "/a.png", Type: "image"}, {URL: "/empty", Type: ""}, {URL: "", Type: "image"}, + }}) + if result.Prompt != "use @图片2 and @视频1" || result.Requirements.Image != 2 || result.Requirements.Video != 1 { + t.Fatalf("unexpected result %#v", result) + } + if len(result.Materials) != 3 || result.Materials[0].Label != "@图片1" || result.Materials[1].Label != "@图片2" || result.Materials[2].Label != "@视频1" { + t.Fatalf("materials %#v", result.Materials) + } + if len(result.Warnings) != 0 { + t.Fatalf("warnings %#v", result.Warnings) + } +} + +func TestAssembleDefaultsImageAndVideoScenes(t *testing.T) { + image := Assemble(Input{Mode: "image", ProjectName: " 项目 ", AspectRatio: "16:9"}) + if image.Blocked || len(image.Scenes) != 5 || image.Prompt == "" || image.Requirements.Image != 4 { + t.Fatalf("image %#v", image) + } + video := Assemble(Input{Mode: "video", ProjectName: "项目"}) + if len(video.Scenes) != 5 || video.Requirements.Image != 5 { + t.Fatalf("video %#v", video) + } +} + +func TestExtractRequirementsDeduplicatesAliases(t *testing.T) { + requirements := ExtractRequirements("@图2 @图片2 @参考视频3 @视频1 @音频4 @图片0") + if requirements.Image != 2 || requirements.Video != 3 || requirements.Audio != 4 { + t.Fatalf("requirements %#v", requirements) + } +} diff --git a/backend/internal/providers/adapters.go b/backend/internal/providers/adapters.go new file mode 100644 index 0000000..ea06279 --- /dev/null +++ b/backend/internal/providers/adapters.go @@ -0,0 +1,223 @@ +package providers + +import ( + "context" + "fmt" + "net/http" + "strings" +) + +type EvoLink struct{ *httpAdapter } + +func NewEvoLink(c Config, client HTTPClient) *EvoLink { + if client == nil { + client = http.DefaultClient + } + a := &httpAdapter{name: "evolink", config: c, client: client, submitPath: func(Request) string { return "/v1/images/generations" }, queryPath: func(id string) string { return "/v1/tasks/" + id }, payload: func(r Request) any { + payload := map[string]any{"model": requestModel(r, c.Model), "prompt": r.Prompt, "n": 1, "resolution": "1K"} + if len(r.InputURLs) > 0 { + payload["image_urls"] = r.InputURLs + } + if quality := strings.TrimSpace(stringValue(r.Settings["quality"])); quality != "" { + payload["quality"] = quality + } + if size := strings.TrimSpace(stringValue(r.Settings["size"])); size != "" { + payload["size"] = size + } else if width, widthOK := positiveInteger(r.Settings["width"]); widthOK { + if height, heightOK := positiveInteger(r.Settings["height"]); heightOK { + payload["size"] = supportedRatio(width, height) + } + } + return payload + }, decode: decodeEvoLink} + return &EvoLink{a} +} +func (a *EvoLink) Submit(c context.Context, r Request) (Result, error) { return a.submit(c, r) } +func (a *EvoLink) Query(c context.Context, id string) (Result, error) { return a.query(c, id) } +func decodeEvoLink(raw []byte) Result { + r := record(raw) + d := object(r["data"]) + out := []string{} + for _, v := range []any{r["results"], d["results"], d["images"], d["image_urls"], d["output"], d["outputs"]} { + collectURLs(v, &out) + } + return Result{TaskID: stringValue(r["id"], r["task_id"], d["id"], d["task_id"]), Status: status(first(r["status"], d["status"])), OutputURLs: out} +} + +type Bailian struct{ *httpAdapter } + +func NewBailian(c Config, client HTTPClient) *Bailian { + if client == nil { + client = http.DefaultClient + } + a := &httpAdapter{name: "bailian", config: c, client: client, submitPath: func(r Request) string { + if r.Capability == "video.generate" { + return "/api/v1/services/aigc/video-generation/video-synthesis" + } + return "/api/v1/services/aigc/image-generation/generation" + }, queryPath: func(id string) string { return "/api/v1/tasks/" + id }, payload: func(r Request) any { + if r.Capability == "video.generate" { + media := make([]any, 0, len(r.InputURLs)) + for index, inputURL := range r.InputURLs { + frameType := "first_frame" + if index > 0 { + frameType = "last_frame" + } + media = append(media, map[string]any{"type": frameType, "url": inputURL}) + } + parameters := map[string]any{ + "resolution": strings.ToUpper(stringValue(r.Settings["resolution"])), + "duration": r.Settings["duration"], + "prompt_extend": true, + "watermark": false, + } + if parameters["resolution"] == "" { + parameters["resolution"] = "720P" + } + if parameters["duration"] == nil { + parameters["duration"] = 10 + } + return map[string]any{"model": requestModel(r, c.Model), "input": map[string]any{"prompt": r.Prompt, "media": media}, "parameters": parameters} + } + content := make([]any, 0, len(r.InputURLs)+1) + for _, inputURL := range r.InputURLs { + content = append(content, map[string]any{"image": inputURL}) + } + content = append(content, map[string]any{"text": r.Prompt}) + parameters := map[string]any{"size": "2K", "n": 1, "watermark": false} + if width, widthOK := positiveInteger(r.Settings["width"]); widthOK { + if height, heightOK := positiveInteger(r.Settings["height"]); heightOK { + parameters["size"] = fmt.Sprintf("%d*%d", width, height) + } + } + if len(r.InputURLs) == 0 { + parameters["thinking_mode"] = true + } + return map[string]any{"model": requestModel(r, c.Model), "input": map[string]any{"messages": []any{map[string]any{"role": "user", "content": content}}}, "parameters": parameters} + }, headers: func(r *http.Request) { r.Header.Set("X-DashScope-Async", "enable") }, decode: decodeBailian} + return &Bailian{a} +} +func (a *Bailian) Submit(c context.Context, r Request) (Result, error) { return a.submit(c, r) } +func (a *Bailian) Query(c context.Context, id string) (Result, error) { return a.query(c, id) } +func decodeBailian(raw []byte) Result { + r := record(raw) + o := object(r["output"]) + out := []string{} + collectURLs(o["results"], &out) + if choices, ok := o["choices"].([]any); ok { + for _, choice := range choices { + message := object(object(choice)["message"]) + if content, ok := message["content"].([]any); ok { + for _, item := range content { + collectURLs(object(item)["image"], &out) + } + } + } + } + collectURLs(o["video_url"], &out) + return Result{TaskID: stringValue(o["task_id"], r["task_id"]), Status: status(o["task_status"]), OutputURLs: out, ErrorMessage: stringValue(r["message"])} +} + +type Seedance struct{ *httpAdapter } + +func NewSeedance(c Config, client HTTPClient) *Seedance { + if client == nil { + client = http.DefaultClient + } + a := &httpAdapter{name: "seedance", config: c, client: client, submitPath: func(Request) string { return "/contents/generations/tasks" }, queryPath: func(id string) string { return "/contents/generations/tasks/" + id }, payload: func(r Request) any { + content := []any{map[string]any{"type": "text", "text": r.Prompt}} + materials := r.Materials + if len(materials) == 0 { + materials = make([]Material, 0, len(r.InputURLs)) + for _, inputURL := range r.InputURLs { + materials = append(materials, Material{URL: inputURL, Type: MaterialImage}) + } + } + for _, material := range materials { + materialType, urlKey, role := "image_url", "image_url", "reference_image" + switch material.Type { + case MaterialVideo: + materialType, urlKey, role = "video_url", "video_url", "reference_video" + case MaterialAudio: + materialType, urlKey, role = "audio_url", "audio_url", "reference_audio" + } + item := map[string]any{"type": materialType, urlKey: map[string]any{"url": material.URL}, "role": role} + if material.Label != "" { + item["label"] = material.Label + } + content = append(content, item) + } + p := map[string]any{"model": requestModel(r, c.Model), "content": content, "generate_audio": true, "watermark": false} + for k, v := range r.Settings { + p[k] = v + } + return p + }, decode: decodeSeedance} + return &Seedance{a} +} +func (a *Seedance) Submit(c context.Context, r Request) (Result, error) { return a.submit(c, r) } +func (a *Seedance) Query(c context.Context, id string) (Result, error) { return a.query(c, id) } +func decodeSeedance(raw []byte) Result { + r := record(raw) + d := object(r["data"]) + content := object(r["content"]) + if len(content) == 0 { + content = object(d["content"]) + } + out := []string{} + for _, v := range []any{content, r["video_url"], r["url"], r["output"], d} { + collectURLs(v, &out) + } + usage := object(r["usage"]) + if len(usage) == 0 { + usage = object(d["usage"]) + } + u := map[string]int{} + if n, ok := usage["completion_tokens"].(float64); ok && n > 0 { + u["completionTokens"] = int(n) + } + return Result{TaskID: stringValue(r["id"], r["task_id"], d["id"], d["task_id"]), Status: status(first(r["status"], d["status"])), OutputURLs: out, ErrorMessage: stringValue(object(r["error"])["message"], object(d["error"])["message"]), Usage: u} +} + +func first(values ...any) any { + for _, v := range values { + if v != nil { + return v + } + } + return nil +} + +func positiveInteger(value any) (int, bool) { + switch number := value.(type) { + case int: + return number, number > 0 + case int64: + return int(number), number > 0 + case float64: + integer := int(number) + return integer, number > 0 && float64(integer) == number + default: + return 0, false + } +} + +func supportedRatio(width, height int) string { + divisor := greatestCommonDivisor(width, height) + ratio := fmt.Sprintf("%d:%d", width/divisor, height/divisor) + switch ratio { + case "1:1", "1:2", "2:1", "1:3", "3:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "9:21", "21:9": + return ratio + default: + return fmt.Sprintf("%dx%d", width, height) + } +} + +func greatestCommonDivisor(left, right int) int { + for right != 0 { + left, right = right, left%right + } + return left +} + +var _ = fmt.Sprint diff --git a/backend/internal/providers/mock.go b/backend/internal/providers/mock.go new file mode 100644 index 0000000..4561ad7 --- /dev/null +++ b/backend/internal/providers/mock.go @@ -0,0 +1,18 @@ +package providers + +import ( + "context" + "crypto/sha256" + "encoding/hex" +) + +type Mock struct{ seed string } + +func NewMock(seed string) *Mock { return &Mock{seed: seed} } +func (m *Mock) Submit(_ context.Context, r Request) (Result, error) { + sum := sha256.Sum256([]byte(m.seed + "\x00" + r.Capability + "\x00" + r.Prompt)) + return Result{TaskID: "mock-" + hex.EncodeToString(sum[:8]), Status: StatusQueued}, nil +} +func (m *Mock) Query(_ context.Context, id string) (Result, error) { + return Result{TaskID: id, Status: StatusSucceeded, OutputURLs: []string{"/generated-results/" + id}}, nil +} diff --git a/backend/internal/providers/provider.go b/backend/internal/providers/provider.go new file mode 100644 index 0000000..21fec33 --- /dev/null +++ b/backend/internal/providers/provider.go @@ -0,0 +1,230 @@ +// Package providers contains bounded protocol adapters for generation services. +package providers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +type Status string + +const ( + StatusQueued Status = "queued" + StatusRunning Status = "running" + StatusSucceeded Status = "succeeded" + StatusFailed Status = "failed" + StatusCancelled Status = "cancelled" + StatusExpired Status = "expired" +) + +type Request struct { + Capability string `json:"capability"` + Model string `json:"model,omitempty"` + Prompt string `json:"prompt"` + InputURLs []string `json:"inputUrls,omitempty"` + Materials []Material `json:"materials,omitempty"` + Settings map[string]any `json:"settings,omitempty"` +} + +type MaterialType string + +const ( + MaterialImage MaterialType = "image" + MaterialVideo MaterialType = "video" + MaterialAudio MaterialType = "audio" +) + +// Material retains the provider-facing type metadata that InputURLs cannot +// express. InputURLs remains supported for existing callers. +type Material struct { + URL string `json:"url"` + Type MaterialType `json:"type"` + Role string `json:"role,omitempty"` + Label string `json:"label,omitempty"` +} + +func requestModel(request Request, fallback string) string { + if value := strings.TrimSpace(request.Model); value != "" { + return value + } + return fallback +} + +type Result struct { + TaskID string + Status Status + OutputURLs []string + Raw json.RawMessage + ErrorMessage string + Usage map[string]int +} + +// HTTPResult is the provider-neutral representation persisted with a Job. +// Output URLs and usage must survive a process restart after the external +// provider has already reached a terminal state. +type HTTPResult struct { + TaskID string `json:"taskId,omitempty"` + Status Status `json:"status"` + OutputURLs []string `json:"outputUrls"` + Raw json.RawMessage `json:"raw,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + Usage map[string]int `json:"usage,omitempty"` +} + +func EncodeResult(result Result) (json.RawMessage, error) { + urls := result.OutputURLs + if urls == nil { + urls = []string{} + } + return json.Marshal(HTTPResult{TaskID: result.TaskID, Status: result.Status, OutputURLs: urls, Raw: result.Raw, ErrorMessage: result.ErrorMessage, Usage: result.Usage}) +} + +type Adapter interface { + Submit(context.Context, Request) (Result, error) + Query(context.Context, string) (Result, error) +} + +// ModelQueryAdapter is implemented by providers whose query protocol requires +// the same model identifier that was used when the task was submitted. Callers +// can opt into it without widening the common Adapter contract. +type ModelQueryAdapter interface { + QueryModel(context.Context, string, string) (Result, error) +} +type HTTPClient interface { + Do(*http.Request) (*http.Response, error) +} +type Config struct { + BaseURL, APIKey, Model, AccessKeyID, SecretAccessKey, Region, Service string + MaxResponseBytes int64 +} + +type ProviderError struct { + Operation string + Status int +} + +func (e *ProviderError) Error() string { + if e.Status > 0 { + return fmt.Sprintf("provider %s failed with HTTP %d", e.Operation, e.Status) + } + return "provider " + e.Operation + " failed" +} + +type httpAdapter struct { + name string + config Config + client HTTPClient + submitPath func(Request) string + queryPath func(string) string + payload func(Request) any + headers func(*http.Request) + decode func([]byte) Result +} + +func (a *httpAdapter) submit(ctx context.Context, input Request) (Result, error) { + body, err := json.Marshal(a.payload(input)) + if err != nil { + return Result{}, fmt.Errorf("encode provider request: %w", err) + } + return a.call(ctx, http.MethodPost, a.submitPath(input), body, "submit") +} +func (a *httpAdapter) query(ctx context.Context, id string) (Result, error) { + if strings.TrimSpace(id) == "" { + return Result{}, errors.New("provider task id is required") + } + return a.call(ctx, http.MethodGet, a.queryPath(url.PathEscape(id)), nil, "query") +} +func (a *httpAdapter) call(ctx context.Context, method, path string, body []byte, operation string) (Result, error) { + base, err := url.Parse(strings.TrimRight(a.config.BaseURL, "/")) + if err != nil { + return Result{}, errors.New("invalid provider base URL") + } + rel, err := url.Parse(path) + if err != nil { + return Result{}, errors.New("invalid provider path") + } + base.Path = strings.TrimRight(base.Path, "/") + "/" + rel.Path = strings.TrimLeft(rel.Path, "/") + target := base.ResolveReference(rel) + req, err := http.NewRequestWithContext(ctx, method, target.String(), strings.NewReader(string(body))) + if err != nil { + return Result{}, fmt.Errorf("build provider request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+a.config.APIKey) + if a.headers != nil { + a.headers(req) + } + resp, err := a.client.Do(req) + if err != nil { + return Result{}, &ProviderError{Operation: a.name + " " + operation} + } + defer resp.Body.Close() + limit := a.config.MaxResponseBytes + if limit <= 0 { + limit = 2 << 20 + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil || int64(len(raw)) > limit { + return Result{}, &ProviderError{Operation: a.name + " " + operation} + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return Result{}, &ProviderError{Operation: a.name + " " + operation, Status: resp.StatusCode} + } + if !json.Valid(raw) { + return Result{}, &ProviderError{Operation: a.name + " " + operation} + } + result := a.decode(raw) + result.Raw = append(json.RawMessage(nil), raw...) + return result, nil +} + +func record(raw []byte) map[string]any { var v map[string]any; _ = json.Unmarshal(raw, &v); return v } +func object(v any) map[string]any { x, _ := v.(map[string]any); return x } +func stringValue(values ...any) string { + for _, v := range values { + if s, ok := v.(string); ok && strings.TrimSpace(s) != "" { + return strings.TrimSpace(s) + } + } + return "" +} +func status(v any) Status { + s := strings.ToLower(stringValue(v)) + switch s { + case "completed", "complete", "succeeded", "success", "done": + return StatusSucceeded + case "running", "processing", "generating", "in_progress": + return StatusRunning + case "failed", "error", "unknown": + return StatusFailed + case "cancelled", "canceled": + return StatusCancelled + case "expired", "not_found", "timeout": + return StatusExpired + default: + return StatusQueued + } +} +func collectURLs(v any, out *[]string) { + switch x := v.(type) { + case string: + if strings.HasPrefix(x, "http://") || strings.HasPrefix(x, "https://") { + *out = append(*out, x) + } + case []any: + for _, i := range x { + collectURLs(i, out) + } + case map[string]any: + for _, k := range []string{"url", "image_url", "imageUrl", "result_url", "resultUrl", "video_url", "file_url"} { + collectURLs(x[k], out) + } + } +} diff --git a/backend/internal/providers/providers_test.go b/backend/internal/providers/providers_test.go new file mode 100644 index 0000000..2e61fd5 --- /dev/null +++ b/backend/internal/providers/providers_test.go @@ -0,0 +1,306 @@ +package providers + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) Do(r *http.Request) (*http.Response, error) { return f(r) } + +func TestHTTPAdaptersMapRequestsAndResponses(t *testing.T) { + tests := []struct { + name string + adapter Adapter + wantSubmitPath, wantQueryPath, response string + }{ + {"evolink", NewEvoLink(Config{BaseURL: "https://e.test", APIKey: "secret", Model: "gpt-image-2"}, nil), "/v1/images/generations", "/v1/tasks/task-1", `{"id":"task-1","status":"completed","results":[{"url":"https://cdn.test/a.png"}]}`}, + {"bailian", NewBailian(Config{BaseURL: "https://b.test", APIKey: "secret", Model: "wan"}, nil), "/api/v1/services/aigc/image-generation/generation", "/api/v1/tasks/task-1", `{"output":{"task_id":"task-1","task_status":"SUCCEEDED","results":[{"url":"https://cdn.test/a.png"}]}}`}, + {"seedance", NewSeedance(Config{BaseURL: "https://s.test/api/v3", APIKey: "secret", Model: "seed"}, nil), "/api/v3/contents/generations/tasks", "/api/v3/contents/generations/tasks/task-1", `{"id":"task-1","status":"succeeded","content":{"video_url":"https://cdn.test/a.mp4"}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var calls int + client := roundTripFunc(func(r *http.Request) (*http.Response, error) { + calls++ + want := tt.wantSubmitPath + if calls == 2 { + want = tt.wantQueryPath + } + if r.URL.Path != want { + t.Fatalf("path=%s want=%s", r.URL.Path, want) + } + if !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ") { + t.Fatal("missing bearer") + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(tt.response)), Header: http.Header{}}, nil + }) + switch a := tt.adapter.(type) { + case *EvoLink: + a.client = client + case *Bailian: + a.client = client + case *Seedance: + a.client = client + } + submitted, err := tt.adapter.Submit(context.Background(), Request{Capability: "image.generate", Prompt: "hello"}) + if err != nil || submitted.TaskID != "task-1" { + t.Fatalf("Submit=%#v,%v", submitted, err) + } + queried, err := tt.adapter.Query(context.Background(), "task-1") + if err != nil || queried.Status != "succeeded" || len(queried.OutputURLs) != 1 { + t.Fatalf("Query=%#v,%v", queried, err) + } + }) + } +} + +func TestVolcengineSignsSubmitAndMapsResult(t *testing.T) { + var request *http.Request + client := roundTripFunc(func(r *http.Request) (*http.Response, error) { + request = r + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"data":{"task_id":"task-1","status":"done","image_urls":["https://cdn.test/a.png"]}}`)), Header: http.Header{}}, nil + }) + a := NewVolcengine(Config{BaseURL: "https://visual.test", AccessKeyID: "ak", SecretAccessKey: "sk", Region: "cn-north-1", Service: "cv", Model: "jimeng"}, client, func() time.Time { return time.Date(2026, 8, 13, 0, 0, 0, 0, time.UTC) }) + got, err := a.Submit(context.Background(), Request{Capability: "image.generate", Prompt: "hello"}) + if err != nil || got.TaskID != "task-1" { + t.Fatalf("Submit=%#v,%v", got, err) + } + if request.URL.Query().Get("Action") != "CVSync2AsyncSubmitTask" || !strings.Contains(request.Header.Get("Authorization"), "Credential=ak/") || request.Header.Get("X-Date") != "20260813T000000Z" { + t.Fatalf("request=%#v headers=%#v", request.URL, request.Header) + } +} + +func TestVolcenginePayloadsMatchJimengSubmitAndQueryProtocols(t *testing.T) { + var bodies []map[string]any + client := roundTripFunc(func(request *http.Request) (*http.Response, error) { + var body map[string]any + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + bodies = append(bodies, body) + response := `{"data":{"task_id":"task-1","status":"queued"}}` + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(response)), Header: http.Header{}}, nil + }) + adapter := NewVolcengine(Config{BaseURL: "https://visual.test", AccessKeyID: "ak", SecretAccessKey: "sk", Model: "jimeng"}, client, func() time.Time { return time.Date(2026, 8, 13, 0, 0, 0, 0, time.UTC) }) + if _, err := adapter.Submit(context.Background(), Request{Prompt: "draw", InputURLs: []string{"https://cdn.test/ref.png"}, Settings: map[string]any{"width": 1024, "height": 768, "force_single": true, "ignored": "value"}}); err != nil { + t.Fatal(err) + } + if _, err := adapter.Query(context.Background(), "task-1"); err != nil { + t.Fatal(err) + } + if bodies[0]["width"] != float64(1024) || bodies[0]["height"] != float64(768) || bodies[0]["force_single"] != true || bodies[0]["ignored"] != nil { + t.Fatalf("submit body=%#v", bodies[0]) + } + queryJSON, ok := bodies[1]["req_json"].(string) + if !ok || queryJSON == "" { + t.Fatalf("query body=%#v", bodies[1]) + } + var queryOptions map[string]any + if err := json.Unmarshal([]byte(queryJSON), &queryOptions); err != nil { + t.Fatal(err) + } + logo := queryOptions["logo_info"].(map[string]any) + if queryOptions["return_url"] != true || logo["add_logo"] != false || logo["opacity"] != float64(1) { + t.Fatalf("query options=%#v", queryOptions) + } +} + +func TestVolcengineQueryUsesPersistedTaskModelInsteadOfAdapterFallback(t *testing.T) { + var body map[string]any + client := roundTripFunc(func(request *http.Request) (*http.Response, error) { + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"data":{"task_id":"task-1","status":"queued"}}`)), Header: http.Header{}}, nil + }) + adapter := NewVolcengine(Config{BaseURL: "https://visual.test", AccessKeyID: "ak", SecretAccessKey: "sk", Model: "fallback-model-b"}, client, func() time.Time { + return time.Date(2026, 8, 13, 0, 0, 0, 0, time.UTC) + }) + + if _, err := adapter.QueryModel(context.Background(), "task-1", "persisted-model-a"); err != nil { + t.Fatal(err) + } + if body["req_key"] != "persisted-model-a" || body["task_id"] != "task-1" { + t.Fatalf("query body=%#v", body) + } +} + +func TestProviderErrorsAreGenericAndDoNotLeakSecrets(t *testing.T) { + client := roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 401, Body: io.NopCloser(strings.NewReader(`{"message":"secret upstream detail"}`)), Header: http.Header{}}, nil + }) + a := NewEvoLink(Config{BaseURL: "https://e.test", APIKey: "very-secret", Model: "m"}, client) + _, err := a.Submit(context.Background(), Request{Capability: "image.generate", Prompt: "p"}) + if err == nil || strings.Contains(err.Error(), "secret") { + t.Fatalf("error=%v", err) + } +} + +func TestBailianUsesThePreparedRequestModelForImageAndVideo(t *testing.T) { + models := []string{} + client := roundTripFunc(func(r *http.Request) (*http.Response, error) { + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + models = append(models, body["model"].(string)) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"output":{"task_id":"task-1","task_status":"PENDING"}}`)), Header: http.Header{}}, nil + }) + adapter := NewBailian(Config{BaseURL: "https://b.test", APIKey: "secret", Model: "fallback-model"}, client) + if _, err := adapter.Submit(context.Background(), Request{Capability: "image.generate", Model: "image-model", Prompt: "image"}); err != nil { + t.Fatal(err) + } + if _, err := adapter.Submit(context.Background(), Request{Capability: "video.generate", Model: "video-model", Prompt: "video"}); err != nil { + t.Fatal(err) + } + if len(models) != 2 || models[0] != "image-model" || models[1] != "video-model" { + t.Fatalf("models=%#v", models) + } +} + +func TestBailianPayloadsMatchImageAndVideoProtocols(t *testing.T) { + tests := []struct { + name string + request Request + check func(*testing.T, map[string]any) + }{ + { + name: "image messages and parameters", + request: Request{Capability: "image.generate", Model: "wan-image", Prompt: "draw", InputURLs: []string{"https://cdn.test/ref.png"}, Settings: map[string]any{ + "width": 1024, "height": 768, + }}, + check: func(t *testing.T, body map[string]any) { + input := body["input"].(map[string]any) + messages := input["messages"].([]any) + content := messages[0].(map[string]any)["content"].([]any) + parameters := body["parameters"].(map[string]any) + if len(content) != 2 || content[0].(map[string]any)["image"] != "https://cdn.test/ref.png" || content[1].(map[string]any)["text"] != "draw" { + t.Fatalf("content=%#v", content) + } + if parameters["size"] != "1024*768" || parameters["n"] != float64(1) || parameters["watermark"] != false { + t.Fatalf("parameters=%#v", parameters) + } + if _, exists := parameters["thinking_mode"]; exists { + t.Fatalf("editing parameters=%#v", parameters) + } + }, + }, + { + name: "video first and last frames", + request: Request{Capability: "video.generate", Model: "wan-video", Prompt: "move", InputURLs: []string{"https://cdn.test/first.png", "https://cdn.test/last.png"}, Settings: map[string]any{ + "resolution": "1080p", "duration": 8, + }}, + check: func(t *testing.T, body map[string]any) { + input := body["input"].(map[string]any) + media := input["media"].([]any) + parameters := body["parameters"].(map[string]any) + if len(media) != 2 || media[0].(map[string]any)["type"] != "first_frame" || media[1].(map[string]any)["type"] != "last_frame" { + t.Fatalf("media=%#v", media) + } + if parameters["resolution"] != "1080P" || parameters["duration"] != float64(8) || parameters["prompt_extend"] != true || parameters["watermark"] != false { + t.Fatalf("parameters=%#v", parameters) + } + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := roundTripFunc(func(request *http.Request) (*http.Response, error) { + var body map[string]any + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + test.check(t, body) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"output":{"task_id":"task-1","task_status":"PENDING"}}`)), Header: http.Header{}}, nil + }) + if _, err := NewBailian(Config{BaseURL: "https://b.test", APIKey: "secret"}, client).Submit(context.Background(), test.request); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestBailianDecodesCompatibleModeChoiceImages(t *testing.T) { + result := decodeBailian([]byte(`{"output":{"task_id":"task-1","task_status":"SUCCEEDED","choices":[{"message":{"content":[{"image":"https://cdn.test/choice.png"}]}}]}}`)) + if result.Status != StatusSucceeded || len(result.OutputURLs) != 1 || result.OutputURLs[0] != "https://cdn.test/choice.png" { + t.Fatalf("result=%#v", result) + } +} + +func TestSeedancePreservesTypedMultimodalMaterials(t *testing.T) { + client := roundTripFunc(func(request *http.Request) (*http.Response, error) { + var body map[string]any + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + content := body["content"].([]any) + if len(content) != 4 { + t.Fatalf("content=%#v", content) + } + checks := []struct{ materialType, urlKey, role string }{ + {"image_url", "image_url", "reference_image"}, + {"video_url", "video_url", "reference_video"}, + {"audio_url", "audio_url", "reference_audio"}, + } + for index, check := range checks { + item := content[index+1].(map[string]any) + if item["type"] != check.materialType || item["role"] != check.role || item["label"] != []string{"图片1", "视频1", "音频1"}[index] { + t.Fatalf("content[%d]=%#v", index+1, item) + } + if object := item[check.urlKey].(map[string]any); object["url"] == "" { + t.Fatalf("content[%d]=%#v", index+1, item) + } + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"id":"task-1","status":"queued"}`)), Header: http.Header{}}, nil + }) + request := Request{ + Capability: "video.generate", Prompt: "combine", + Materials: []Material{ + {URL: "https://cdn.test/image.png", Type: MaterialImage, Label: "图片1"}, + {URL: "https://cdn.test/video.mp4", Type: MaterialVideo, Label: "视频1"}, + {URL: "https://cdn.test/audio.mp3", Type: MaterialAudio, Label: "音频1"}, + }, + } + if _, err := NewSeedance(Config{BaseURL: "https://s.test/api/v3", APIKey: "secret", Model: "seed"}, client).Submit(context.Background(), request); err != nil { + t.Fatal(err) + } +} + +func TestEvoLinkPayloadIncludesQualityAndSize(t *testing.T) { + client := roundTripFunc(func(request *http.Request) (*http.Response, error) { + var body map[string]any + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if body["quality"] != "high" || body["size"] != "16:9" || body["resolution"] != "1K" { + t.Fatalf("body=%#v", body) + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"id":"task-1","status":"queued"}`)), Header: http.Header{}}, nil + }) + _, err := NewEvoLink(Config{BaseURL: "https://e.test", APIKey: "secret", Model: "gpt-image-2"}, client).Submit(context.Background(), Request{ + Capability: "image.generate", Prompt: "draw", Settings: map[string]any{"quality": " high ", "width": 1920, "height": 1080}, + }) + if err != nil { + t.Fatal(err) + } +} + +func TestMockIsDeterministic(t *testing.T) { + m := NewMock("fixture") + a, _ := m.Submit(context.Background(), Request{Capability: "video.generate", Prompt: "hello"}) + b, _ := m.Submit(context.Background(), Request{Capability: "video.generate", Prompt: "hello"}) + if a.TaskID != b.TaskID { + t.Fatalf("task IDs differ: %s %s", a.TaskID, b.TaskID) + } + got, _ := m.Query(context.Background(), a.TaskID) + if got.Status != "succeeded" || len(got.OutputURLs) != 1 { + t.Fatalf("Query=%#v", got) + } +} diff --git a/backend/internal/providers/volcengine.go b/backend/internal/providers/volcengine.go new file mode 100644 index 0000000..7e84817 --- /dev/null +++ b/backend/internal/providers/volcengine.go @@ -0,0 +1,125 @@ +package providers + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +type Volcengine struct { + config Config + client HTTPClient + now func() time.Time +} + +func NewVolcengine(c Config, client HTTPClient, now func() time.Time) *Volcengine { + if client == nil { + client = http.DefaultClient + } + if now == nil { + now = time.Now + } + if c.Region == "" { + c.Region = "cn-north-1" + } + if c.Service == "" { + c.Service = "cv" + } + return &Volcengine{c, client, now} +} +func (v *Volcengine) Submit(ctx context.Context, r Request) (Result, error) { + p := map[string]any{"req_key": requestModel(r, v.config.Model), "prompt": r.Prompt, "image_urls": r.InputURLs} + for _, key := range []string{"scale", "width", "height", "min_ratio", "max_ratio", "force_single"} { + if value, exists := r.Settings[key]; exists && value != nil && value != "" { + p[key] = value + } + } + return v.call(ctx, "CVSync2AsyncSubmitTask", p) +} +func (v *Volcengine) Query(ctx context.Context, id string) (Result, error) { + return v.QueryModel(ctx, id, v.config.Model) +} +func (v *Volcengine) QueryModel(ctx context.Context, id, model string) (Result, error) { + queryOptions, _ := json.Marshal(map[string]any{ + "return_url": true, + "logo_info": map[string]any{"add_logo": false, "position": 0, "language": 0, "opacity": 1}, + }) + return v.call(ctx, "CVSync2AsyncGetResult", map[string]any{"req_key": requestModel(Request{Model: model}, v.config.Model), "task_id": id, "req_json": string(queryOptions)}) +} +func (v *Volcengine) call(ctx context.Context, action string, payload any) (Result, error) { + body, _ := json.Marshal(payload) + endpoint, err := url.Parse(v.config.BaseURL) + if err != nil { + return Result{}, errors.New("invalid provider base URL") + } + q := endpoint.Query() + q.Set("Action", action) + q.Set("Version", "2022-08-31") + endpoint.RawQuery = canonicalQuery(q) + date := v.now().UTC() + xdate := date.Format("20060102T150405Z") + short := xdate[:8] + hash := sha(body) + headers := "content-type:application/json\nhost:" + endpoint.Host + "\nx-content-sha256:" + hash + "\nx-date:" + xdate + "\n" + signed := "content-type;host;x-content-sha256;x-date" + canonicalPath := endpoint.EscapedPath() + if canonicalPath == "" { + canonicalPath = "/" + } + canonical := "POST\n" + canonicalPath + "\n" + endpoint.RawQuery + "\n" + headers + "\n" + signed + "\n" + hash + scope := short + "/" + v.config.Region + "/" + v.config.Service + "/request" + stringToSign := "HMAC-SHA256\n" + xdate + "\n" + scope + "\n" + sha([]byte(canonical)) + key := hmacBytes([]byte(v.config.SecretAccessKey), short) + key = hmacBytes(key, v.config.Region) + key = hmacBytes(key, v.config.Service) + key = hmacBytes(key, "request") + signature := hex.EncodeToString(hmacBytes(key, stringToSign)) + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), strings.NewReader(string(body))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Content-Sha256", hash) + req.Header.Set("X-Date", xdate) + req.Header.Set("Authorization", "HMAC-SHA256 Credential="+v.config.AccessKeyID+"/"+scope+", SignedHeaders="+signed+", Signature="+signature) + resp, err := v.client.Do(req) + if err != nil { + return Result{}, &ProviderError{Operation: "volcengine request"} + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 || !json.Valid(raw) { + return Result{}, &ProviderError{Operation: "volcengine request", Status: resp.StatusCode} + } + r := record(raw) + d := object(r["data"]) + out := []string{} + collectURLs(d, &out) + return Result{TaskID: stringValue(r["task_id"], d["task_id"]), Status: status(first(r["status"], d["status"])), OutputURLs: out, Raw: raw}, nil +} +func sha(b []byte) string { x := sha256.Sum256(b); return hex.EncodeToString(x[:]) } +func hmacBytes(k []byte, s string) []byte { + h := hmac.New(sha256.New, k) + _, _ = h.Write([]byte(s)) + return h.Sum(nil) +} +func canonicalQuery(q url.Values) string { + keys := make([]string, 0, len(q)) + for k := range q { + keys = append(keys, k) + } + sort.Strings(keys) + parts := []string{} + for _, k := range keys { + for _, v := range q[k] { + parts = append(parts, url.QueryEscape(k)+"="+url.QueryEscape(v)) + } + } + return strings.Join(parts, "&") +} diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.go new file mode 100644 index 0000000..f386a59 --- /dev/null +++ b/backend/internal/settings/service.go @@ -0,0 +1,496 @@ +package settings + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" +) + +type RuntimeUpdater func(context.Context, map[string]string) error +type Option struct { + Label string `json:"label"` + Value string `json:"value"` +} +type Field struct { + Key string `json:"key"` + Label string `json:"label"` + Description string `json:"description,omitempty"` + Secret bool `json:"secret,omitempty"` + Type string `json:"type,omitempty"` + Options []Option `json:"options,omitempty"` + Value string `json:"value"` + Configured bool `json:"configured"` + DefaultValue string `json:"-"` +} +type Group struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Fields []Field `json:"fields"` +} +type Capability struct { + ID string `json:"id"` + Label string `json:"label"` + ReqKey string `json:"reqKey"` + Engine string `json:"engine,omitempty"` + EngineLabel string `json:"engineLabel,omitempty"` + Enabled bool `json:"enabled"` +} +type EngineAssignment struct { + ID string `json:"id"` + Label string `json:"label"` + Engine string `json:"engine"` + EngineLabel string `json:"engineLabel"` + Connected bool `json:"connected"` + ConnectionLabel string `json:"connectionLabel"` + ReqKey string `json:"reqKey"` + Configurable bool `json:"configurable"` + Field *Field `json:"field,omitempty"` +} +type Services struct { + Visual bool `json:"visual"` + Evolink bool `json:"evolink"` + Seedance bool `json:"seedance"` + Bailian bool `json:"bailian"` + Auth bool `json:"auth"` + Organization bool `json:"organization"` +} +type Payload struct { + Services Services `json:"services"` + Capabilities []Capability `json:"capabilities"` + EngineAssignments []EngineAssignment `json:"engineAssignments"` + Groups []Group `json:"groups"` + RestartRequired bool `json:"restartRequired,omitempty"` +} +type Service struct { + mu sync.Mutex + path string + environment map[string]string + update RuntimeUpdater + billing BillingAccountWriter +} + +type BillingAccount struct { + AccountName, BankName, AccountNumber, Contact string +} + +type BillingAccountWriter interface { + SaveBillingAccount(context.Context, BillingAccount) error +} + +func New(path string, environment map[string]string, updater RuntimeUpdater) *Service { + return &Service{path: path, environment: cloneStrings(environment), update: updater} +} + +// LoadEnvironment reads only the settings whitelist. Values already supplied +// by the process win, matching Next's process.env-over-file precedence. +func LoadEnvironment(path string, environment map[string]string) (map[string]string, error) { + file, err := readEnv(path) + if err != nil { + return nil, err + } + merged := cloneStrings(environment) + for key, value := range file { + if _, allowed := fieldIndex[key]; !allowed { + continue + } + if _, exists := merged[key]; !exists { + merged[key] = value + } + } + return merged, nil +} + +func RuntimeSettingKeys() []string { + keys := make([]string, 0, len(fieldIndex)) + for key := range fieldIndex { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func (s *Service) WithBillingAccountWriter(writer BillingAccountWriter) *Service { + s.billing = writer + return s +} +func (s *Service) Get(ctx context.Context) (any, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + s.mu.Lock() + defer s.mu.Unlock() + return s.get() +} +func (s *Service) Save(ctx context.Context, values map[string]any) (any, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + s.mu.Lock() + defer s.mu.Unlock() + updates := map[string]string{} + for key, raw := range values { + field, ok := fieldIndex[key] + next, stringValue := raw.(string) + if !ok || !stringValue { + continue + } + next = strings.TrimSpace(next) + if field.Secret && next == "" { + continue + } + updates[key] = next + } + if len(updates) > 0 { + // Capture the complete persisted view before writing so a partial billing + // account update cannot blank sibling fields that only exist in the file. + persisted, err := readEnv(s.path) + if err != nil { + return nil, err + } + if err := s.write(updates); err != nil { + return nil, err + } + if s.update != nil { + if err := s.update(ctx, cloneStrings(updates)); err != nil { + return nil, fmt.Errorf("apply runtime settings: %w", err) + } + } + for key, value := range updates { + s.environment[key] = value + persisted[key] = value + } + if s.billing != nil && containsBillingAccountUpdate(updates) { + if err := s.billing.SaveBillingAccount(ctx, BillingAccount{ + AccountName: billingSetting(s.environment, persisted, "ZHINIAN_BILLING_ACCOUNT_NAME"), + BankName: billingSetting(s.environment, persisted, "ZHINIAN_BILLING_ACCOUNT_BANK"), + AccountNumber: billingSetting(s.environment, persisted, "ZHINIAN_BILLING_ACCOUNT_NUMBER"), + Contact: billingSetting(s.environment, persisted, "ZHINIAN_BILLING_CONTACT"), + }); err != nil { + return nil, fmt.Errorf("apply billing account settings: %w", err) + } + } + } + payload, err := s.get() + if err == nil && requiresRestart(updates) { + payload.RestartRequired = true + } + return payload, err +} + +func billingSetting(environment, persisted map[string]string, key string) string { + if value, ok := environment[key]; ok { + return value + } + return persisted[key] +} + +func containsBillingAccountUpdate(updates map[string]string) bool { + for key := range updates { + switch key { + case "ZHINIAN_BILLING_ACCOUNT_NAME", "ZHINIAN_BILLING_ACCOUNT_BANK", "ZHINIAN_BILLING_ACCOUNT_NUMBER", "ZHINIAN_BILLING_CONTACT": + return true + } + } + return false +} + +func requiresRestart(updates map[string]string) bool { + for key := range updates { + switch key { + case "ZHINIAN_BILLING_ACCOUNT_NAME", "ZHINIAN_BILLING_ACCOUNT_BANK", "ZHINIAN_BILLING_ACCOUNT_NUMBER", "ZHINIAN_BILLING_CONTACT": + continue + default: + return true + } + } + return false +} +func (s *Service) get() (Payload, error) { + file, err := readEnv(s.path) + if err != nil { + return Payload{}, err + } + current := func(field Field) string { + if value, ok := s.environment[field.Key]; ok { + return value + } + if value, ok := file[field.Key]; ok { + return value + } + return field.DefaultValue + } + groups := definitions() + for gi := range groups { + for fi := range groups[gi].Fields { + field := &groups[gi].Fields[fi] + raw := current(*field) + field.Configured = raw != "" + if !field.Secret { + field.Value = raw + } + } + } + image := normalizeImage(current(fieldIndex["IMAGE_GENERATE_ENGINE"])) + video := normalizeVideo(current(fieldIndex["VIDEO_GENERATE_ENGINE"])) + imageModel := map[string]string{"jimeng": lookup(s.environment, file, "JIMENG_IMAGE_GENERATE_46_REQ_KEY", "jimeng_seedream46_cvtob"), "evolink": lookup(s.environment, file, "EVOLINK_IMAGE_MODEL", "gpt-image-2"), "bailian": lookup(s.environment, file, "BAILIAN_IMAGE_MODEL", "wan2.7-image-pro")}[image] + videoModel := lookup(s.environment, file, "SEEDANCE_MODEL", "doubao-seedance-2-0-260128") + if video == "bailian" { + videoModel = lookup(s.environment, file, "BAILIAN_VIDEO_MODEL", "wan2.7-i2v-2026-04-25") + } + imageConnected := connected(image, s.environment, file) + videoConnected := connected(video, s.environment, file) + imageField := project(fieldIndex["IMAGE_GENERATE_ENGINE"], image, s.environment, file) + videoField := project(fieldIndex["VIDEO_GENERATE_ENGINE"], video, s.environment, file) + assignments := []EngineAssignment{{ID: "image.generate", Label: "图片生成", Engine: image, EngineLabel: label(image), Connected: imageConnected, ConnectionLabel: connection(imageConnected), ReqKey: imageModel, Configurable: true, Field: &imageField}, {ID: "video.generate", Label: "视频生成", Engine: video, EngineLabel: label(video), Connected: videoConnected, ConnectionLabel: connection(videoConnected), ReqKey: videoModel, Configurable: true, Field: &videoField}} + services := Services{Visual: connected("jimeng", s.environment, file), Evolink: connected("evolink", s.environment, file), Seedance: connected("seedance", s.environment, file), Bailian: connected("bailian", s.environment, file), Auth: lookup(s.environment, file, "ZHINIAN_AUTH_SESSION_SECRET", "") != "", Organization: lookup(s.environment, file, "DATABASE_URL", "") != "" || lookup(s.environment, file, "ZHINIAN_AUTH_SESSION_SECRET", "") != ""} + return Payload{Services: services, Capabilities: []Capability{{ID: "image.generate", Label: "图片生成 4.6", ReqKey: imageModel, Engine: image, EngineLabel: label(image), Enabled: true}, {ID: "video.generate", Label: "视频生成", ReqKey: videoModel, Engine: video, EngineLabel: label(video), Enabled: true}}, EngineAssignments: assignments, Groups: groups}, nil +} +func (s *Service) write(updates map[string]string) error { + data, err := os.ReadFile(s.path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("read settings file: %w", err) + } + if errors.Is(err, os.ErrNotExist) { + data = []byte("# 智念AIGC平台 API 配置\n") + } + lines := strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n") + seen := map[string]bool{} + for index, line := range lines { + key, _, ok := splitLine(line) + if !ok { + continue + } + if value, replace := updates[key]; replace { + lines[index] = key + "=" + formatValue(value) + seen[key] = true + } + } + missing := []string{} + for key := range updates { + if !seen[key] { + missing = append(missing, key) + } + } + sort.Strings(missing) + if len(missing) > 0 { + for len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + if len(lines) > 0 { + lines = append(lines, "") + } + lines = append(lines, "# Managed by 智念AIGC平台 设置页") + for _, key := range missing { + lines = append(lines, key+"="+formatValue(updates[key])) + } + } + if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil { + return err + } + temp, err := os.CreateTemp(filepath.Dir(s.path), ".settings-*.tmp") + if err != nil { + return err + } + name := temp.Name() + defer os.Remove(name) + if err = temp.Chmod(0o600); err == nil { + _, err = temp.WriteString(strings.TrimRight(strings.Join(lines, "\n"), "\n") + "\n") + } + if err == nil { + err = temp.Sync() + } + closeErr := temp.Close() + if err == nil { + err = closeErr + } + if err != nil { + return err + } + return os.Rename(name, s.path) +} +func readEnv(path string) (map[string]string, error) { + file, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return map[string]string{}, nil + } + if err != nil { + return nil, err + } + defer file.Close() + result := map[string]string{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + key, raw, ok := splitLine(scanner.Text()) + if ok { + result[key] = parseValue(raw) + } + } + return result, scanner.Err() +} +func splitLine(line string) (string, string, bool) { + index := strings.IndexByte(line, '=') + if index < 1 { + return "", "", false + } + key := strings.TrimSpace(line[:index]) + if !validKey(key) { + return "", "", false + } + return key, strings.TrimSpace(line[index+1:]), true +} +func validKey(key string) bool { + for index, r := range key { + if !(r == '_' || r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' || index > 0 && r >= '0' && r <= '9') { + return false + } + } + return key != "" +} +func parseValue(raw string) string { + raw = strings.TrimSpace(raw) + if len(raw) >= 2 && ((raw[0] == '"' && raw[len(raw)-1] == '"') || (raw[0] == '\'' && raw[len(raw)-1] == '\'')) { + if raw[0] == '\'' { + return raw[1 : len(raw)-1] + } + var out strings.Builder + escaped := false + for _, r := range raw[1 : len(raw)-1] { + if escaped { + switch r { + case 'n': + out.WriteByte('\n') + case '"', '\\': + out.WriteRune(r) + default: + out.WriteByte('\\') + out.WriteRune(r) + } + escaped = false + } else if r == '\\' { + escaped = true + } else { + out.WriteRune(r) + } + } + return out.String() + } + if i := strings.Index(raw, " #"); i >= 0 { + raw = raw[:i] + } + return strings.TrimSpace(raw) +} +func formatValue(value string) string { + if value == "" { + return "" + } + if !strings.ContainsAny(value, " \t\r\n#\"'\\") { + return value + } + return "\"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"", "\n", "\\n", "\r", "\\r").Replace(value) + "\"" +} +func cloneStrings(input map[string]string) map[string]string { + out := map[string]string{} + for key, value := range input { + out[key] = value + } + return out +} +func lookup(environment, file map[string]string, key, fallback string) string { + if value, ok := environment[key]; ok { + return value + } + if value, ok := file[key]; ok { + return value + } + return fallback +} +func project(field Field, value string, environment, file map[string]string) Field { + field.Value = value + field.Configured = lookup(environment, file, field.Key, "") != "" + return field +} +func normalizeImage(value string) string { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "evolink" || value == "bailian" { + return value + } + return "jimeng" +} +func normalizeVideo(value string) string { + if strings.ToLower(strings.TrimSpace(value)) == "bailian" { + return "bailian" + } + return "seedance" +} +func label(engine string) string { + switch engine { + case "evolink": + return "EvoLink" + case "bailian": + return "阿里云百炼" + case "seedance": + return "Seedance" + default: + return "即梦" + } +} +func connection(ok bool) string { + if ok { + return "已连接" + } + return "待配置" +} +func connected(engine string, environment, file map[string]string) bool { + switch engine { + case "evolink": + return mock(lookup(environment, file, "EVOLINK_MOCK", "auto"), lookup(environment, file, "EVOLINK_API_KEY", "") != "") + case "seedance": + return mock(lookup(environment, file, "SEEDANCE_MOCK", "auto"), lookup(environment, file, "SEEDANCE_API_KEY", "") != "") + case "bailian": + flag := strings.ToLower(strings.TrimSpace(lookup(environment, file, "BAILIAN_MOCK", "auto"))) + return flag != "1" && flag != "true" && lookup(environment, file, "BAILIAN_API_KEY", "") != "" + default: + return mock(lookup(environment, file, "JIMENG_VISUAL_MOCK", "auto"), lookup(environment, file, "VOLCENGINE_ACCESS_KEY_ID", "") != "" && lookup(environment, file, "VOLCENGINE_SECRET_ACCESS_KEY", "") != "") + } +} +func mock(flag string, configured bool) bool { + flag = strings.ToLower(strings.TrimSpace(flag)) + if flag == "1" || flag == "true" { + return false + } + if flag == "0" || flag == "false" { + return true + } + return configured +} + +func definitions() []Group { + return []Group{ + {ID: "auth", Title: "平台账号安全", Description: "平台自建手机号账号登录。", Fields: []Field{{Key: "ZHINIAN_AUTH_REQUIRED", Label: "登录保护", Type: "select", DefaultValue: "auto", Options: []Option{{Label: "自动", Value: "auto"}, {Label: "启用", Value: "1"}, {Label: "停用", Value: "0"}}}, {Key: "ZHINIAN_AUTH_SESSION_SECRET", Label: "会话签名密钥", Secret: true, Type: "password"}}}, + {ID: "billing", Title: "企业计费", Description: "企业计费和对公账户信息。", Fields: []Field{{Key: "ZHINIAN_BILLING_REQUIRED", Label: "真实任务计费", Type: "select", DefaultValue: "1", Options: []Option{{Label: "启用", Value: "1"}, {Label: "停用(免计费)", Value: "0"}}}, {Key: "ZHINIAN_BILLING_ACCOUNT_NAME", Label: "对公账户名称"}, {Key: "ZHINIAN_BILLING_ACCOUNT_BANK", Label: "开户行"}, {Key: "ZHINIAN_BILLING_ACCOUNT_NUMBER", Label: "银行账号"}, {Key: "ZHINIAN_BILLING_CONTACT", Label: "充值对接信息"}}}, + {ID: "visual", Title: "即梦图片 API", Description: "火山 AK/SK。", Fields: []Field{{Key: "VOLCENGINE_ACCESS_KEY_ID", Label: "Access Key ID", Secret: true, Type: "password"}, {Key: "VOLCENGINE_SECRET_ACCESS_KEY", Label: "Secret Access Key", Secret: true, Type: "password"}}}, + {ID: "evolink", Title: "EvoLink 图片 API", Description: "GPT Image 2 图片生成。", Fields: []Field{{Key: "EVOLINK_API_KEY", Label: "EvoLink API Key", Secret: true, Type: "password"}, {Key: "EVOLINK_BASE_URL", Label: "Base URL", DefaultValue: "https://api.evolink.ai"}, {Key: "EVOLINK_IMAGE_MODEL", Label: "图片模型", DefaultValue: "gpt-image-2"}, {Key: "EVOLINK_IMAGE_QUALITY", Label: "质量", DefaultValue: "medium"}}}, + {ID: "seedance", Title: "Seedance 视频 API", Description: "火山方舟 API Key。", Fields: []Field{{Key: "SEEDANCE_API_KEY", Label: "方舟 API Key", Secret: true, Type: "password"}}}, + {ID: "bailian", Title: "阿里云百炼 API", Description: "万相图片与视频。", Fields: []Field{{Key: "BAILIAN_API_KEY", Label: "百炼 API Key", Secret: true, Type: "password"}, {Key: "BAILIAN_BASE_URL", Label: "Base URL", DefaultValue: "https://llm-126wneubbdo6dbr5.cn-beijing.maas.aliyuncs.com/compatible-mode/v1"}, {Key: "BAILIAN_IMAGE_MODEL", Label: "图片模型", DefaultValue: "wan2.7-image-pro"}, {Key: "BAILIAN_VIDEO_MODEL", Label: "视频模型", DefaultValue: "wan2.7-i2v-2026-04-25"}}}, + {ID: "oss", Title: "OSS 资产存储", Description: "共享资产存储。", Fields: []Field{{Key: "ALI_OSS_ENDPOINT", Label: "Endpoint"}, {Key: "ALI_OSS_BUCKET", Label: "Bucket"}, {Key: "ALI_OSS_ACCESS_KEY_ID", Label: "Access Key ID", Secret: true, Type: "password"}, {Key: "ALI_OSS_ACCESS_KEY_SECRET", Label: "Access Key Secret", Secret: true, Type: "password"}, {Key: "ALI_OSS_PUBLIC_BASE_URL", Label: "公开访问 Base URL"}}}, + } +} + +var fieldIndex = func() map[string]Field { + result := map[string]Field{} + for _, group := range definitions() { + for _, field := range group.Fields { + result[field.Key] = field + } + } + result["IMAGE_GENERATE_ENGINE"] = Field{Key: "IMAGE_GENERATE_ENGINE", Label: "图片生成", Type: "select", DefaultValue: "jimeng", Options: []Option{{Label: "即梦 / 火山视觉", Value: "jimeng"}, {Label: "EvoLink GPT Image 2", Value: "evolink"}, {Label: "阿里云百炼 Wan 2.7", Value: "bailian"}}} + result["VIDEO_GENERATE_ENGINE"] = Field{Key: "VIDEO_GENERATE_ENGINE", Label: "视频生成", Type: "select", DefaultValue: "bailian", Options: []Option{{Label: "Seedance", Value: "seedance"}, {Label: "阿里云百炼 Wan 2.7", Value: "bailian"}}} + return result +}() diff --git a/backend/internal/settings/service_test.go b/backend/internal/settings/service_test.go new file mode 100644 index 0000000..3a40510 --- /dev/null +++ b/backend/internal/settings/service_test.go @@ -0,0 +1,180 @@ +package settings + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestFixtureIsConsumedByGo(t *testing.T) { + data, err := os.ReadFile("../../../contracts/settings/runtime-v1.json") + if err != nil { + t.Fatal(err) + } + var fixture struct { + Version int `json:"version"` + FileName string `json:"fileName"` + AllowedKeys []string `json:"allowedKeys"` + SecretKeys []string `json:"secretKeys"` + } + if err := json.Unmarshal(data, &fixture); err != nil { + t.Fatal(err) + } + if fixture.Version != 1 || fixture.FileName != ".env.local" || len(fixture.AllowedKeys) != 25 || len(fixture.SecretKeys) != 8 { + t.Fatalf("fixture = %#v", fixture) + } +} + +func TestServiceSavePreservesSecretsAndUnrelatedEnvLines(t *testing.T) { + path := filepath.Join(t.TempDir(), ".env.local") + original := "# operator note\nDATABASE_URL=postgres://untouched\nZHINIAN_AUTH_SESSION_SECRET=keep-me\nZHINIAN_BILLING_CONTACT=old # prior\nCUSTOM_QUOTED=\"a b\"\n" + if err := os.WriteFile(path, []byte(original), 0o600); err != nil { + t.Fatal(err) + } + environment := map[string]string{"EVOLINK_BASE_URL": "https://runtime.example"} + var applied map[string]string + service := New(path, environment, func(_ context.Context, updates map[string]string) error { applied = clone(updates); return nil }) + + value, err := service.Save(context.Background(), map[string]any{ + "ZHINIAN_AUTH_SESSION_SECRET": " ", + "ZHINIAN_BILLING_CONTACT": " ", + "EVOLINK_BASE_URL": " https://new.example/v1 ", + "DATABASE_URL": "postgres://attack", + "ALI_OSS_BUCKET": 42, + }) + if err != nil { + t.Fatal(err) + } + text, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + got := string(text) + for _, expected := range []string{"# operator note", "DATABASE_URL=postgres://untouched", "ZHINIAN_AUTH_SESSION_SECRET=keep-me", "ZHINIAN_BILLING_CONTACT=", "CUSTOM_QUOTED=\"a b\"", "EVOLINK_BASE_URL=https://new.example/v1"} { + if !strings.Contains(got, expected) { + t.Fatalf("file missing %q:\n%s", expected, got) + } + } + if strings.Contains(got, "postgres://attack") || strings.Contains(got, "ALI_OSS_BUCKET=42") { + t.Fatalf("unexpected unapproved update:\n%s", got) + } + if !reflect.DeepEqual(applied, map[string]string{"ZHINIAN_BILLING_CONTACT": "", "EVOLINK_BASE_URL": "https://new.example/v1"}) { + t.Fatalf("applied = %#v", applied) + } + if payload := value.(Payload); !payload.RestartRequired { + t.Fatal("provider setting update must declare restartRequired") + } + assertSecretProjection(t, value, "ZHINIAN_AUTH_SESSION_SECRET", true) +} + +func TestServiceGetUsesInjectedEnvironmentWithoutMutatingProcess(t *testing.T) { + path := filepath.Join(t.TempDir(), ".env.local") + if err := os.WriteFile(path, []byte("EVOLINK_BASE_URL=file-value\nSEEDANCE_API_KEY=file-secret\n"), 0o600); err != nil { + t.Fatal(err) + } + const sentinel = "settings-service-must-not-write-this" + old, present := os.LookupEnv(sentinel) + t.Cleanup(func() { + if present { + _ = os.Setenv(sentinel, old) + } else { + _ = os.Unsetenv(sentinel) + } + }) + _ = os.Unsetenv(sentinel) + service := New(path, map[string]string{"EVOLINK_BASE_URL": "runtime-value", "SEEDANCE_API_KEY": "runtime-secret"}, nil) + value, err := service.Get(context.Background()) + if err != nil { + t.Fatal(err) + } + if _, exists := os.LookupEnv(sentinel); exists { + t.Fatal("process environment mutated") + } + if got := fieldValue(t, value, "EVOLINK_BASE_URL"); got != "runtime-value" { + t.Fatalf("value = %q", got) + } + assertSecretProjection(t, value, "SEEDANCE_API_KEY", true) +} + +func TestServiceSynchronizesBillingAccountWithoutRequiringRestart(t *testing.T) { + writer := &billingWriterStub{} + service := New(filepath.Join(t.TempDir(), ".env.local"), nil, nil).WithBillingAccountWriter(writer) + value, err := service.Save(context.Background(), map[string]any{ + "ZHINIAN_BILLING_ACCOUNT_NAME": " Acme ", + "ZHINIAN_BILLING_ACCOUNT_BANK": " Bank ", + "ZHINIAN_BILLING_ACCOUNT_NUMBER": " 123 ", + "ZHINIAN_BILLING_CONTACT": " Ops ", + }) + if err != nil { + t.Fatal(err) + } + if writer.value != (BillingAccount{AccountName: "Acme", BankName: "Bank", AccountNumber: "123", Contact: "Ops"}) { + t.Fatalf("billing account=%#v", writer.value) + } + if value.(Payload).RestartRequired { + t.Fatal("billing account update should be applied immediately") + } +} + +func TestServicePartialBillingUpdatePreservesPersistedSiblingFields(t *testing.T) { + path := filepath.Join(t.TempDir(), ".env.local") + if err := os.WriteFile(path, []byte("ZHINIAN_BILLING_ACCOUNT_NAME=Acme\nZHINIAN_BILLING_ACCOUNT_BANK=Legacy Bank\nZHINIAN_BILLING_ACCOUNT_NUMBER=123\nZHINIAN_BILLING_CONTACT=Old Ops\n"), 0o600); err != nil { + t.Fatal(err) + } + writer := &billingWriterStub{} + service := New(path, nil, nil).WithBillingAccountWriter(writer) + if _, err := service.Save(context.Background(), map[string]any{"ZHINIAN_BILLING_CONTACT": "New Ops"}); err != nil { + t.Fatal(err) + } + want := BillingAccount{AccountName: "Acme", BankName: "Legacy Bank", AccountNumber: "123", Contact: "New Ops"} + if writer.value != want { + t.Fatalf("billing account=%#v want %#v", writer.value, want) + } +} + +type billingWriterStub struct{ value BillingAccount } + +func (writer *billingWriterStub) SaveBillingAccount(_ context.Context, value BillingAccount) error { + writer.value = value + return nil +} + +func fieldValue(t *testing.T, value any, key string) string { + t.Helper() + payload := value.(Payload) + for _, group := range payload.Groups { + for _, field := range group.Fields { + if field.Key == key { + return field.Value + } + } + } + t.Fatalf("field %s not found", key) + return "" +} +func assertSecretProjection(t *testing.T, value any, key string, configured bool) { + t.Helper() + payload := value.(Payload) + for _, group := range payload.Groups { + for _, field := range group.Fields { + if field.Key == key { + if field.Value != "" || field.Configured != configured { + t.Fatalf("field = %#v", field) + } + return + } + } + } + t.Fatalf("field %s not found", key) +} +func clone(input map[string]string) map[string]string { + output := map[string]string{} + for key, value := range input { + output[key] = value + } + return output +} diff --git a/backend/internal/templates/service.go b/backend/internal/templates/service.go new file mode 100644 index 0000000..97a98f6 --- /dev/null +++ b/backend/internal/templates/service.go @@ -0,0 +1,250 @@ +// Package templates owns user-scoped image-template validation and persistence. +package templates + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "math" + "net/url" + "strings" + "time" +) + +var ( + ErrInvalidTemplate = errors.New("invalid image template") + ErrNotFound = errors.New("image template not found") +) + +type Settings struct { + Engine string `json:"engine,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + ForceSingle *bool `json:"forceSingle,omitempty"` + Scale float64 `json:"scale,omitempty"` + Quality string `json:"quality,omitempty"` +} + +type Template struct { + ID string `json:"id"` + OwnerID string `json:"ownerId"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Prompt string `json:"prompt"` + PreviewImageURL string `json:"previewImageUrl,omitempty"` + Settings Settings `json:"settings"` + SortOrder int `json:"sortOrder"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type CreateCommand struct { + Name, Description, Prompt, PreviewImageURL string + Settings Settings + SortOrder int +} + +type UpdateCommand struct { + Name, Description, Prompt, PreviewImageURL *string + Settings *Settings + SortOrder *int +} + +type Patch = UpdateCommand + +type Catalog interface { + ListTemplates(context.Context, string) ([]Template, error) + CreateTemplate(context.Context, Template) (Template, error) + UpdateTemplate(context.Context, string, string, Patch, time.Time) (Template, bool, error) + DeleteTemplate(context.Context, string, string) (Template, bool, error) +} + +type Service struct { + catalog Catalog + now func() time.Time + newID func() string +} + +func NewService(catalog Catalog, now func() time.Time, newID func() string) *Service { + if now == nil { + now = time.Now + } + if newID == nil { + newID = templateID + } + return &Service{catalog: catalog, now: now, newID: newID} +} + +func (service *Service) List(ctx context.Context, owner string) ([]Template, error) { + if strings.TrimSpace(owner) == "" { + return nil, fmt.Errorf("%w: owner is required", ErrInvalidTemplate) + } + items, err := service.catalog.ListTemplates(ctx, owner) + if err != nil { + return nil, fmt.Errorf("list image templates: %w", err) + } + if items == nil { + items = []Template{} + } + return items, nil +} + +func (service *Service) Create(ctx context.Context, owner string, command CreateCommand) (Template, error) { + owner = strings.TrimSpace(owner) + if owner == "" { + return Template{}, fmt.Errorf("%w: owner is required", ErrInvalidTemplate) + } + name, err := required(command.Name, 80, "模板名称") + if err != nil { + return Template{}, err + } + prompt, err := required(command.Prompt, 4000, "预设提示词") + if err != nil { + return Template{}, err + } + preview, err := previewURL(command.PreviewImageURL) + if err != nil { + return Template{}, err + } + settings := normalizeSettings(command.Settings) + now := service.now().UTC() + item := Template{ID: service.newID(), OwnerID: owner, Name: name, Description: optional(command.Description, 240), Prompt: prompt, PreviewImageURL: preview, Settings: settings, SortOrder: command.SortOrder, CreatedAt: now, UpdatedAt: now} + created, err := service.catalog.CreateTemplate(ctx, item) + if err != nil { + return Template{}, fmt.Errorf("create image template: %w", err) + } + return created, nil +} + +func (service *Service) Update(ctx context.Context, owner, id string, command UpdateCommand) (Template, error) { + owner, id = strings.TrimSpace(owner), strings.TrimSpace(id) + if owner == "" || id == "" { + return Template{}, ErrNotFound + } + patch := Patch{} + var err error + if command.Name != nil { + value, current := required(*command.Name, 80, "模板名称") + err = current + patch.Name = &value + } + if err != nil { + return Template{}, err + } + if command.Prompt != nil { + value, current := required(*command.Prompt, 4000, "预设提示词") + err = current + patch.Prompt = &value + } + if err != nil { + return Template{}, err + } + if command.Description != nil { + value := optional(*command.Description, 240) + patch.Description = &value + } + if command.PreviewImageURL != nil { + value, current := previewURL(*command.PreviewImageURL) + if current != nil { + return Template{}, current + } + patch.PreviewImageURL = &value + } + if command.Settings != nil { + value := normalizeSettings(*command.Settings) + patch.Settings = &value + } + patch.SortOrder = command.SortOrder + item, found, err := service.catalog.UpdateTemplate(ctx, owner, id, patch, service.now().UTC()) + if err != nil { + return Template{}, fmt.Errorf("update image template: %w", err) + } + if !found { + return Template{}, ErrNotFound + } + return item, nil +} + +func (service *Service) Delete(ctx context.Context, owner, id string) (Template, error) { + item, found, err := service.catalog.DeleteTemplate(ctx, strings.TrimSpace(owner), strings.TrimSpace(id)) + if err != nil { + return Template{}, fmt.Errorf("delete image template: %w", err) + } + if !found { + return Template{}, ErrNotFound + } + return item, nil +} + +func applyPatch(item *Template, patch Patch) { + if patch.Name != nil { + item.Name = *patch.Name + } + if patch.Description != nil { + item.Description = *patch.Description + } + if patch.Prompt != nil { + item.Prompt = *patch.Prompt + } + if patch.PreviewImageURL != nil { + item.PreviewImageURL = *patch.PreviewImageURL + } + if patch.Settings != nil { + item.Settings = *patch.Settings + } + if patch.SortOrder != nil { + item.SortOrder = *patch.SortOrder + } +} + +func required(value string, limit int, label string) (string, error) { + value = optional(value, limit) + if value == "" { + return "", fmt.Errorf("%w: %s不能为空", ErrInvalidTemplate, label) + } + return value, nil +} +func optional(value string, limit int) string { + value = strings.TrimSpace(value) + if len([]rune(value)) > limit { + value = string([]rune(value)[:limit]) + } + return value +} +func previewURL(value string) (string, error) { + value = optional(value, 1000) + if value == "" || strings.HasPrefix(value, "/") { + return value, nil + } + parsed, err := url.Parse(value) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return "", fmt.Errorf("%w: 效果预览图地址必须是 http(s) 或站内路径", ErrInvalidTemplate) + } + return value, nil +} +func normalizeSettings(value Settings) Settings { + if value.Engine != "jimeng" && value.Engine != "evolink" && value.Engine != "bailian" { + value.Engine = "" + } + if value.Width < 1 || value.Height < 1 { + value.Width, value.Height = 0, 0 + } else { + value.Width, value.Height = min(value.Width, 8192), min(value.Height, 8192) + } + if math.IsNaN(value.Scale) || math.IsInf(value.Scale, 0) { + value.Scale = 0 + } else if value.Scale != 0 { + value.Scale = math.Max(1, math.Min(100, math.Trunc(value.Scale))) + } + if value.Quality != "low" && value.Quality != "medium" && value.Quality != "high" { + value.Quality = "" + } + return value +} +func templateID() string { + var raw [9]byte + _, _ = rand.Read(raw[:]) + return "tmpl_" + hex.EncodeToString(raw[:]) +} diff --git a/backend/internal/templates/service_test.go b/backend/internal/templates/service_test.go new file mode 100644 index 0000000..d0bc18c --- /dev/null +++ b/backend/internal/templates/service_test.go @@ -0,0 +1,116 @@ +package templates + +import ( + "context" + "errors" + "testing" + "time" +) + +type memoryCatalog struct { + items []Template + err error +} + +func (catalog *memoryCatalog) ListTemplates(_ context.Context, owner string) ([]Template, error) { + if catalog.err != nil { + return nil, catalog.err + } + out := []Template{} + for _, item := range catalog.items { + if item.OwnerID == owner { + out = append(out, item) + } + } + return out, nil +} +func (catalog *memoryCatalog) CreateTemplate(_ context.Context, item Template) (Template, error) { + if catalog.err != nil { + return Template{}, catalog.err + } + catalog.items = append(catalog.items, item) + return item, nil +} +func (catalog *memoryCatalog) UpdateTemplate(_ context.Context, owner, id string, patch Patch, now time.Time) (Template, bool, error) { + if catalog.err != nil { + return Template{}, false, catalog.err + } + for index := range catalog.items { + if catalog.items[index].OwnerID != owner || catalog.items[index].ID != id { + continue + } + applyPatch(&catalog.items[index], patch) + catalog.items[index].UpdatedAt = now + return catalog.items[index], true, nil + } + return Template{}, false, nil +} +func (catalog *memoryCatalog) DeleteTemplate(_ context.Context, owner, id string) (Template, bool, error) { + if catalog.err != nil { + return Template{}, false, catalog.err + } + for index, item := range catalog.items { + if item.OwnerID != owner || item.ID != id { + continue + } + catalog.items = append(catalog.items[:index], catalog.items[index+1:]...) + return item, true, nil + } + return Template{}, false, nil +} + +func TestServiceNormalizesAndScopesTemplates(t *testing.T) { + now := time.Date(2026, 8, 13, 12, 0, 0, 0, time.UTC) + catalog := &memoryCatalog{} + service := NewService(catalog, func() time.Time { return now }, func() string { return "tmpl_fixed" }) + created, err := service.Create(context.Background(), "owner-a", CreateCommand{ + Name: " Campaign ", Description: " ", Prompt: " make it vivid ", + PreviewImageURL: "/uploads/preview.png", SortOrder: 2, + Settings: Settings{Engine: "jimeng", Width: 2048, Height: 2048, Scale: 50}, + }) + if err != nil { + t.Fatal(err) + } + if created.ID != "tmpl_fixed" || created.OwnerID != "owner-a" || created.Name != "Campaign" || created.Prompt != "make it vivid" { + t.Fatalf("unexpected template: %#v", created) + } + if created.Description != "" || created.CreatedAt != now || created.UpdatedAt != now { + t.Fatalf("unexpected normalized values: %#v", created) + } + if _, err := service.Create(context.Background(), "owner-a", CreateCommand{Name: "x", Prompt: "p", PreviewImageURL: "ftp://bad"}); !errors.Is(err, ErrInvalidTemplate) { + t.Fatalf("expected invalid preview URL, got %v", err) + } + items, err := service.List(context.Background(), "owner-b") + if err != nil || len(items) != 0 { + t.Fatalf("scope leak: %#v %v", items, err) + } +} + +func TestServiceUpdateDeleteAndNotFound(t *testing.T) { + now := time.Now().UTC() + catalog := &memoryCatalog{items: []Template{{ID: "t1", OwnerID: "owner", Name: "old", Prompt: "p", Settings: Settings{}, CreatedAt: now, UpdatedAt: now}}} + service := NewService(catalog, func() time.Time { return now.Add(time.Hour) }, nil) + name := " new " + updated, err := service.Update(context.Background(), "owner", "t1", UpdateCommand{Name: &name}) + if err != nil || updated.Name != "new" { + t.Fatalf("update: %#v %v", updated, err) + } + if _, err := service.Update(context.Background(), "other", "t1", UpdateCommand{Name: &name}); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected scoped not found, got %v", err) + } + deleted, err := service.Delete(context.Background(), "owner", "t1") + if err != nil || deleted.ID != "t1" { + t.Fatalf("delete: %#v %v", deleted, err) + } + if _, err := service.Delete(context.Background(), "owner", "t1"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected not found, got %v", err) + } +} + +func TestServicePropagatesInfrastructureFailures(t *testing.T) { + want := errors.New("database unavailable") + service := NewService(&memoryCatalog{err: want}, nil, nil) + if _, err := service.List(context.Background(), "owner"); !errors.Is(err, want) { + t.Fatalf("expected wrapped cause, got %v", err) + } +} diff --git a/backend/internal/usage/report.go b/backend/internal/usage/report.go new file mode 100644 index 0000000..595136c --- /dev/null +++ b/backend/internal/usage/report.go @@ -0,0 +1,404 @@ +package usage + +import ( + "context" + "crypto/sha256" + "fmt" + "math" + "sort" + "time" + + "golang.org/x/text/collate" + "golang.org/x/text/language" +) + +type ContextRepository interface { + ListContext(context.Context, Filters) ([]Event, error) +} + +func (s Service) Personal(ctx context.Context, request PersonalRequest) (PersonalReport, error) { + rangeValue, err := PresetRange(request.Preset, request.Now) + if err != nil { + return PersonalReport{}, err + } + events, err := s.list(ctx, Filters{OwnerID: request.AccountID, From: rangeValue.From, To: rangeValue.To}) + if err != nil { + return PersonalReport{}, err + } + events = eligible(events) + recent := records(events) + if len(recent) > 5 { + recent = recent[:5] + } + return PersonalReport{Preset: request.Preset, Range: rangeValue, Total: len(events), ByCapability: capabilityCounts(events), Recent: recent}, nil +} +func (s Service) Admin(ctx context.Context, request AdminRequest) (AdminReport, error) { + rangeValue, err := adminRange(request) + if err != nil { + return AdminReport{}, err + } + baseEvents, err := s.list(ctx, Filters{From: rangeValue.From, To: rangeValue.To}) + if err != nil { + return AdminReport{}, err + } + baseEvents = eligible(baseEvents) + injectedOrganizations, err := s.organizationOptions(ctx, request.Requester) + if err != nil { + return AdminReport{}, err + } + baseViews := recordsWithOrganizations(baseEvents, injectedOrganizations) + views := filterRecords(baseViews, request) + events := eventsForRecords(baseEvents, views) + recent := views + if len(recent) > 100 { + recent = recent[:100] + } + optionViews := baseViews + if request.OrganizationID != "" { + optionViews = filterRecords(baseViews, AdminRequest{Requester: Requester{OrganizationID: request.OrganizationID}}) + } + accounts := map[string]bool{} + organizations := map[string]bool{} + for _, e := range events { + accounts[e.OwnerID] = true + if e.OrganizationID != "" { + organizations[e.OrganizationID] = true + } + } + return AdminReport{Range: rangeValue, Summary: Summary{Total: len(events), ActiveAccounts: len(accounts), ActiveOrganizations: len(organizations), AveragePerDay: math.Round(float64(len(events))/float64(rangeValue.DayCount)*10) / 10}, Trend: trend(events, rangeValue), ByCapability: capabilityCounts(events), ByProvider: providerCounts(events), Organizations: organizationRows(views), Accounts: accountRows(views), Recent: recent, Options: Options{Organizations: mergeOrganizationOptions(injectedOrganizations, optionViews), Accounts: accountOptions(optionViews), Capabilities: []Option{{"image.generate", "图片生成"}, {"video.generate", "视频生成"}}, Providers: providerOptions(baseEvents)}}, nil +} + +func (s Service) organizationOptions(ctx context.Context, requester Requester) ([]Option, error) { + if s.OrganizationOptions == nil { + return []Option{}, nil + } + options, err := s.OrganizationOptions.ListOrganizationOptions(ctx, requester) + if options == nil { + options = []Option{} + } + return options, err +} + +func recordsWithOrganizations(events []Event, organizations []Option) []Record { + names := make(map[string]string, len(organizations)) + for _, option := range organizations { + names[option.Value] = option.Label + } + views := records(events) + for i := range views { + if events[i].OrganizationName == "" { + if name := names[views[i].OrganizationID]; name != "" { + views[i].OrganizationName = name + } + } + } + return views +} + +func filterRecords(records []Record, request AdminRequest) []Record { + out := make([]Record, 0, len(records)) + for _, record := range records { + if request.OrganizationID != "" && record.OrganizationID != request.OrganizationID || request.OwnerID != "" && record.OwnerID != request.OwnerID || request.Capability != "" && record.Capability != request.Capability || request.Provider != "" && record.Provider != request.Provider { + continue + } + out = append(out, record) + } + return out +} + +func eventsForRecords(events []Event, records []Record) []Event { + byID := make(map[string]Event, len(events)) + for _, event := range events { + byID[event.ID] = event + } + out := make([]Event, 0, len(records)) + for _, record := range records { + if event, ok := byID[record.ID]; ok { + out = append(out, event) + } + } + return out +} +func (s Service) list(ctx context.Context, filters Filters) ([]Event, error) { + if contextual, ok := s.Repository.(ContextRepository); ok { + return contextual.ListContext(ctx, filters) + } + return s.Repository.List(filters) +} +func eligible(events []Event) []Event { + seen := map[string]bool{} + out := make([]Event, 0, len(events)) + for _, e := range events { + if e.Source == "api" || e.Provider == "mock" || seen[e.JobID] { + continue + } + seen[e.JobID] = true + out = append(out, e) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt > out[j].CreatedAt }) + return out +} +func records(events []Event) []Record { + out := make([]Record, 0, len(events)) + for _, e := range events { + name := e.AccountDisplayName + if name == "" { + name = e.AccountUsername + } + if name == "" { + sum := sha256.Sum256([]byte(e.OwnerID)) + name = fmt.Sprintf("历史账号 %X", sum[:4]) + } + orgName := e.OrganizationName + organizationID := e.OrganizationID + if organizationID == "" { + organizationID = UnassignedOrganizationID + } + if orgName == "" { + orgName = "未归属组织" + } + out = append(out, Record{ID: e.ID, JobID: e.JobID, OwnerID: e.OwnerID, AccountName: name, AccountUsername: e.AccountUsername, OrganizationID: organizationID, OrganizationName: orgName, Capability: e.Capability, CapabilityLabel: capabilityLabel(e.Capability), Provider: e.Provider, ProviderLabel: providerLabel(e.Provider), ReqKey: e.ReqKey, CreatedAt: e.CreatedAt}) + } + return out +} +func capabilityCounts(events []Event) []CountItem { + counts := map[string]int{} + for _, e := range events { + counts[e.Capability]++ + } + return []CountItem{{"image.generate", "图片生成", counts["image.generate"]}, {"video.generate", "视频生成", counts["video.generate"]}} +} +func providerCounts(events []Event) []CountItem { + counts := map[string]int{} + for _, e := range events { + key := e.Provider + if key == "" { + key = "unknown" + } + counts[key]++ + } + out := make([]CountItem, 0, len(counts)) + for key, count := range counts { + out = append(out, CountItem{key, providerLabel(key), count}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Key < out[j].Key + }) + return out +} +func providerOptions(events []Event) []Option { + seen := map[string]bool{} + out := []Option{} + for _, e := range events { + if e.Provider != "" && !seen[e.Provider] { + seen[e.Provider] = true + out = append(out, Option{e.Provider, providerLabel(e.Provider)}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Label < out[j].Label }) + return out +} +func mergeOrganizationOptions(injected []Option, records []Record) []Option { + labels := make(map[string]string, len(injected)+len(records)) + for _, option := range injected { + if option.Value != "" { + labels[option.Value] = option.Label + } + } + for _, record := range records { + id := record.OrganizationID + if id == "" { + id = UnassignedOrganizationID + } + labels[id] = record.OrganizationName + } + out := make([]Option, 0, len(labels)) + for value, label := range labels { + if label == "" { + label = value + } + out = append(out, Option{Value: value, Label: label}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Value == UnassignedOrganizationID { + return false + } + if out[j].Value == UnassignedOrganizationID { + return true + } + if out[i].Label != out[j].Label { + return out[i].Label < out[j].Label + } + return out[i].Value < out[j].Value + }) + return out +} +func accountOptions(records []Record) []Option { + labels := make(map[string]string, len(records)) + for _, record := range records { + if _, exists := labels[record.OwnerID]; exists { + continue + } + label := record.AccountName + if record.AccountUsername != "" { + label += "(" + record.AccountUsername + ")" + } + labels[record.OwnerID] = label + } + out := make([]Option, 0, len(labels)) + for value, label := range labels { + out = append(out, Option{Value: value, Label: label}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Label != out[j].Label { + return out[i].Label < out[j].Label + } + return out[i].Value < out[j].Value + }) + return out +} +func trend(events []Event, r DateRange) []TrendPoint { + counts := map[string]int{} + monthly := r.DayCount > 62 + for _, e := range events { + if parsed, err := time.Parse(time.RFC3339, e.CreatedAt); err == nil { + layout := "2006-01-02" + if monthly { + layout = "2006-01" + } + counts[parsed.In(shanghai).Format(layout)]++ + } + } + out := []TrendPoint{} + if monthly { + for key, count := range counts { + out = append(out, TrendPoint{key, key, count}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Date < out[j].Date }) + return out + } + start, _ := time.ParseInLocation("2006-01-02", r.StartDate, shanghai) + end, _ := time.ParseInLocation("2006-01-02", r.EndDate, shanghai) + for day := start; !day.After(end); day = day.AddDate(0, 0, 1) { + key := day.Format("2006-01-02") + out = append(out, TrendPoint{key, key[5:], counts[key]}) + } + return out +} +func organizationRows(records []Record) any { + by := map[string]*organizationRow{} + for _, v := range records { + id := v.OrganizationID + if id == "" { + id = UnassignedOrganizationID + } + item := by[id] + if item == nil { + item = &organizationRow{OrganizationID: id, OrganizationName: v.OrganizationName, accounts: map[string]bool{}} + by[id] = item + } + item.Count++ + item.accounts[v.OwnerID] = true + item.AccountCount = len(item.accounts) + if v.CreatedAt > item.LastUsedAt { + item.LastUsedAt = v.CreatedAt + } + } + out := make([]organizationRow, 0, len(by)) + for _, v := range by { + out = append(out, *v) + } + zhCN := collate.New(language.Chinese) + sort.SliceStable(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + if out[i].OrganizationID == UnassignedOrganizationID { + return false + } + if out[j].OrganizationID == UnassignedOrganizationID { + return true + } + return zhCN.CompareString(out[i].OrganizationName, out[j].OrganizationName) < 0 + }) + return out +} +func accountRows(records []Record) any { + by := map[string]*accountRow{} + order := make([]string, 0, len(records)) + for _, v := range records { + item := by[v.OwnerID] + if item == nil { + item = &accountRow{OwnerID: v.OwnerID, AccountName: v.AccountName, AccountUsername: v.AccountUsername, OrganizationID: v.OrganizationID, OrganizationName: v.OrganizationName} + by[v.OwnerID] = item + order = append(order, v.OwnerID) + } + item.Count++ + if v.CreatedAt > item.LastUsedAt { + item.LastUsedAt = v.CreatedAt + } + } + out := make([]accountRow, 0, len(by)) + for _, id := range order { + out = append(out, *by[id]) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Count > out[j].Count }) + return out +} +func adminRange(request AdminRequest) (DateRange, error) { + if request.StartDate == "" && request.EndDate == "" { + return PresetRange(PresetMonth, request.Now) + } + fallback, _ := PresetRange(PresetMonth, request.Now) + start, end := request.StartDate, request.EndDate + if start == "" { + start = fallback.StartDate + } + if end == "" { + end = fallback.EndDate + } + from, err := time.ParseInLocation("2006-01-02", start, shanghai) + if err != nil { + return DateRange{}, err + } + to, err := time.ParseInLocation("2006-01-02", end, shanghai) + if err != nil || to.Before(from) { + return DateRange{}, ErrInvalidDateRange + } + dayCount := int(to.Sub(from).Hours()/24) + 1 + if dayCount > 3660 { + return DateRange{}, ErrDateRangeTooLong + } + label := start + if start != end { + label = start + " 至 " + end + } + return DateRange{From: from.Format(time.RFC3339), To: to.AddDate(0, 0, 1).Format(time.RFC3339), StartDate: start, EndDate: end, Label: label, DayCount: dayCount}, nil +} +func capabilityLabel(value string) string { + if value == "video.generate" { + return "视频生成" + } + return "图片生成" +} +func providerLabel(value string) string { + switch value { + case "volcengine-visual": + return "即梦图片" + case "evolink": + return "EvoLink" + case "seedance": + return "Seedance" + case "bailian": + return "阿里云百炼" + case "mock": + return "系统" + default: + return "未知服务商" + } +} + +var _ Reporter = Service{} diff --git a/backend/internal/usage/report_test.go b/backend/internal/usage/report_test.go new file mode 100644 index 0000000..3bc7daf --- /dev/null +++ b/backend/internal/usage/report_test.go @@ -0,0 +1,162 @@ +package usage + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + "time" +) + +func TestPersonalReportFiltersDuplicateAPIAndMockEvents(t *testing.T) { + repo := &reportRepository{events: []Event{{ID: "1", JobID: "job", OwnerID: "user", Source: "platform", Capability: "image.generate", Provider: "bailian", CreatedAt: "2026-08-13T01:00:00Z"}, {ID: "duplicate", JobID: "job", OwnerID: "user", Source: "platform", Capability: "image.generate", Provider: "bailian", CreatedAt: "2026-08-13T00:00:00Z"}, {ID: "api", JobID: "api", OwnerID: "user", Source: "api", Capability: "image.generate", Provider: "bailian", CreatedAt: "2026-08-13T00:00:00Z"}, {ID: "mock", JobID: "mock", OwnerID: "user", Source: "platform", Capability: "image.generate", Provider: "mock", CreatedAt: "2026-08-13T00:00:00Z"}}} + report, err := (Service{Repository: repo}).Personal(context.Background(), PersonalRequest{AccountID: "user", Preset: PresetToday, Now: time.Date(2026, 8, 13, 1, 0, 0, 0, time.UTC)}) + if err != nil { + t.Fatal(err) + } + if report.Total != 1 || len(report.Recent) != 1 || report.ByCapability[0].Count != 1 { + t.Fatalf("report=%+v", report) + } +} + +func TestAdminReportEmitsEmptyOptionArraysInsteadOfNull(t *testing.T) { + report, err := (Service{Repository: &reportRepository{}}).Admin(context.Background(), AdminRequest{Now: time.Date(2026, 8, 13, 1, 0, 0, 0, time.UTC)}) + if err != nil { + t.Fatal(err) + } + if report.Options.Organizations == nil || report.Options.Accounts == nil || report.Options.Providers == nil || report.Trend == nil { + t.Fatalf("report contains nil collection: %#v", report) + } +} + +func TestAdminReportBuildsAccountOptionsFromBaseRangeBeforeReportFilters(t *testing.T) { + repo := &reportRepository{events: []Event{ + {ID: "selected", JobID: "job-selected", OwnerID: "account-a", Source: "platform", Capability: "image.generate", Provider: "bailian", OrganizationID: "org-a", AccountDisplayName: "甲账号", AccountUsername: "13800000001", CreatedAt: "2026-08-12T01:00:00Z"}, + {ID: "same-org", JobID: "job-same-org", OwnerID: "account-b", Source: "platform", Capability: "video.generate", Provider: "seedance", OrganizationID: "org-a", AccountDisplayName: "乙账号", CreatedAt: "2026-08-12T00:00:00Z"}, + {ID: "other-org", JobID: "job-other-org", OwnerID: "account-c", Source: "platform", Capability: "image.generate", Provider: "evolink", OrganizationID: "org-b", AccountDisplayName: "丙账号", CreatedAt: "2026-08-11T00:00:00Z"}, + }} + + report, err := (Service{Repository: repo}).Admin(context.Background(), AdminRequest{ + Requester: Requester{OrganizationID: "org-a"}, + OwnerID: "account-a", + Capability: "image.generate", + Provider: "bailian", + Now: time.Date(2026, 8, 13, 1, 0, 0, 0, time.UTC), + }) + if err != nil { + t.Fatal(err) + } + if report.Summary.Total != 1 { + t.Fatalf("filtered report total = %d, want 1", report.Summary.Total) + } + want := []Option{{Value: "account-b", Label: "乙账号"}, {Value: "account-a", Label: "甲账号(13800000001)"}} + if !reflect.DeepEqual(report.Options.Accounts, want) { + t.Fatalf("account options = %#v, want %#v", report.Options.Accounts, want) + } +} + +func TestAdminReportIncludesInjectedOrganizationsWithoutUsage(t *testing.T) { + source := organizationOptionSourceStub{options: []Option{ + {Value: "org-no-usage", Label: "零用量组织"}, + {Value: "org-used", Label: "组织目录名称"}, + }} + repo := &reportRepository{events: []Event{{ID: "used", JobID: "job-used", OwnerID: "account", Source: "platform", Capability: "image.generate", Provider: "bailian", OrganizationID: "org-used", CreatedAt: "2026-08-12T01:00:00Z"}}} + + report, err := (Service{Repository: repo, OrganizationOptions: source}).Admin(context.Background(), AdminRequest{Now: time.Date(2026, 8, 13, 1, 0, 0, 0, time.UTC)}) + if err != nil { + t.Fatal(err) + } + want := []Option{{Value: "org-used", Label: "组织目录名称"}, {Value: "org-no-usage", Label: "零用量组织"}} + if !reflect.DeepEqual(report.Options.Organizations, want) { + t.Fatalf("organization options = %#v, want %#v", report.Options.Organizations, want) + } + recent := report.Recent.([]Record) + if len(recent) != 1 || recent[0].OrganizationName != "组织目录名称" { + t.Fatalf("recent records = %#v", recent) + } +} + +func TestAdminReportAggregatesLongRangesByMonth(t *testing.T) { + repo := &reportRepository{events: []Event{ + {ID: "jan", JobID: "job-jan", OwnerID: "account", Source: "platform", Capability: "image.generate", Provider: "bailian", CreatedAt: "2026-01-31T16:30:00Z"}, + {ID: "apr", JobID: "job-apr", OwnerID: "account", Source: "platform", Capability: "image.generate", Provider: "bailian", CreatedAt: "2026-04-01T00:00:00Z"}, + }} + report, err := (Service{Repository: repo}).Admin(context.Background(), AdminRequest{StartDate: "2026-01-01", EndDate: "2026-04-30"}) + if err != nil { + t.Fatal(err) + } + want := []TrendPoint{{Date: "2026-02", Label: "2026-02", Count: 1}, {Date: "2026-04", Label: "2026-04", Count: 1}} + if !reflect.DeepEqual(report.Trend, want) { + t.Fatalf("trend = %#v, want %#v", report.Trend, want) + } +} + +func TestAdminReportRejectsRangesLongerThan3660Days(t *testing.T) { + _, err := (Service{Repository: &reportRepository{}}).Admin(context.Background(), AdminRequest{StartDate: "2016-01-01", EndDate: "2026-08-13"}) + if !errors.Is(err, ErrDateRangeTooLong) { + t.Fatalf("error = %v, want ErrDateRangeTooLong", err) + } +} + +func TestAdminReportLimitsRecentAndUsesCanonicalLabels(t *testing.T) { + events := make([]Event, 101) + for i := range events { + provider := "evolink" + if i == 0 { + provider = "volcengine-visual" + } + events[i] = Event{ID: fmt.Sprintf("event-%03d", i), JobID: fmt.Sprintf("job-%03d", i), OwnerID: "account", Source: "platform", Capability: "image.generate", Provider: provider, CreatedAt: fmt.Sprintf("2026-08-12T%02d:%02d:00Z", i/60, i%60)} + } + report, err := (Service{Repository: &reportRepository{events: events}}).Admin(context.Background(), AdminRequest{Now: time.Date(2026, 8, 13, 1, 0, 0, 0, time.UTC)}) + if err != nil { + t.Fatal(err) + } + recent := report.Recent.([]Record) + if len(recent) != 100 { + t.Fatalf("recent length = %d, want 100", len(recent)) + } + if recent[99].ID != "event-001" { + t.Fatalf("last recent id = %q, want event-001", recent[99].ID) + } + if recent[0].OrganizationID != UnassignedOrganizationID { + t.Fatalf("organization id = %q, want %q", recent[0].OrganizationID, UnassignedOrganizationID) + } + if recent[0].ProviderLabel != "EvoLink" || report.Options.Providers[1] != (Option{Value: "volcengine-visual", Label: "即梦图片"}) { + t.Fatalf("provider labels: record=%q options=%#v", recent[0].ProviderLabel, report.Options.Providers) + } +} + +func TestAdminReportIncludesRangeLabelAndDeterministicBreakdowns(t *testing.T) { + repo := &reportRepository{events: []Event{ + {ID: "unassigned", JobID: "job-unassigned", OwnerID: "account-z", AccountDisplayName: "乙账号", Source: "platform", Capability: "image.generate", Provider: "bailian", CreatedAt: "2026-08-12T03:00:00Z"}, + {ID: "org-b", JobID: "job-b", OwnerID: "account-b", AccountDisplayName: "甲账号", OrganizationID: "org-b", OrganizationName: "乙组织", Source: "platform", Capability: "image.generate", Provider: "bailian", CreatedAt: "2026-08-12T02:00:00Z"}, + {ID: "org-a", JobID: "job-a", OwnerID: "account-a", AccountDisplayName: "丙账号", OrganizationID: "org-a", OrganizationName: "甲组织", Source: "platform", Capability: "image.generate", Provider: "bailian", CreatedAt: "2026-08-12T01:00:00Z"}, + }} + report, err := (Service{Repository: repo}).Admin(context.Background(), AdminRequest{StartDate: "2026-08-11", EndDate: "2026-08-12"}) + if err != nil { + t.Fatal(err) + } + if report.Range.Label != "2026-08-11 至 2026-08-12" { + t.Fatalf("range label = %q", report.Range.Label) + } + organizations := report.Organizations.([]organizationRow) + if got := []string{organizations[0].OrganizationID, organizations[1].OrganizationID, organizations[2].OrganizationID}; !reflect.DeepEqual(got, []string{"org-a", "org-b", UnassignedOrganizationID}) { + t.Fatalf("organization order = %#v", got) + } + accounts := report.Accounts.([]accountRow) + if got := []string{accounts[0].OwnerID, accounts[1].OwnerID, accounts[2].OwnerID}; !reflect.DeepEqual(got, []string{"account-z", "account-b", "account-a"}) { + t.Fatalf("account order = %#v", got) + } +} + +type reportRepository struct{ events []Event } + +func (r *reportRepository) Insert(e Event) (Event, bool, error) { return e, true, nil } +func (r *reportRepository) List(Filters) ([]Event, error) { return r.events, nil } + +type organizationOptionSourceStub struct{ options []Option } + +func (s organizationOptionSourceStub) ListOrganizationOptions(context.Context, Requester) ([]Option, error) { + return s.options, nil +} diff --git a/backend/internal/usage/usage.go b/backend/internal/usage/usage.go index d18cded..eba55dc 100644 --- a/backend/internal/usage/usage.go +++ b/backend/internal/usage/usage.go @@ -1,6 +1,7 @@ package usage import ( + "context" "errors" "fmt" "sort" @@ -17,11 +18,20 @@ const ( ) var ErrForbidden = errors.New("usage scope forbidden") +var ErrInvalidDateRange = errors.New("invalid usage date range") +var ErrDateRangeTooLong = errors.New("usage date range exceeds 3660 days") + +const UnassignedOrganizationID = "__unassigned__" + var shanghai = time.FixedZone("Asia/Shanghai", 8*60*60) type DateRange struct { - From, To, StartDate, EndDate string - DayCount int + From string `json:"from"` + To string `json:"to"` + StartDate string `json:"startDate"` + EndDate string `json:"endDate"` + Label string `json:"label"` + DayCount int `json:"dayCount"` } func PresetRange(preset Preset, now time.Time) (DateRange, error) { @@ -39,7 +49,8 @@ func PresetRange(preset Preset, now time.Time) (DateRange, error) { return DateRange{}, fmt.Errorf("unknown usage preset %q", preset) } end := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, shanghai) - return DateRange{From: start.Format(time.RFC3339), To: end.AddDate(0, 0, 1).Format(time.RFC3339), StartDate: start.Format("2006-01-02"), EndDate: end.Format("2006-01-02"), DayCount: int(end.Sub(start).Hours()/24) + 1}, nil + labels := map[Preset]string{PresetToday: "今天", Preset7Days: "近 7 天", Preset30Days: "近 30 天", PresetMonth: "本月"} + return DateRange{From: start.Format(time.RFC3339), To: end.AddDate(0, 0, 1).Format(time.RFC3339), StartDate: start.Format("2006-01-02"), EndDate: end.Format("2006-01-02"), Label: labels[preset], DayCount: int(end.Sub(start).Hours()/24) + 1}, nil } type Event struct { @@ -53,7 +64,111 @@ type Repository interface { Insert(Event) (Event, bool, error) List(Filters) ([]Event, error) } -type Service struct{ Repository Repository } +type OrganizationOptionSource interface { + ListOrganizationOptions(context.Context, Requester) ([]Option, error) +} +type OrganizationOptionSourceFunc func(context.Context, Requester) ([]Option, error) + +func (fn OrganizationOptionSourceFunc) ListOrganizationOptions(ctx context.Context, requester Requester) ([]Option, error) { + return fn(ctx, requester) +} + +type Service struct { + Repository Repository + OrganizationOptions OrganizationOptionSource +} + +type PersonalRequest struct { + AccountID string + Preset Preset + Now time.Time +} +type AdminRequest struct { + Requester + StartDate, EndDate, OwnerID, Capability, Provider string + Now time.Time + RedactAccounts bool +} +type CountItem struct { + Key string `json:"key"` + Label string `json:"label"` + Count int `json:"count"` +} +type Record struct { + ID string `json:"id"` + JobID string `json:"jobId"` + OwnerID string `json:"ownerId"` + AccountName string `json:"accountName"` + AccountUsername string `json:"accountUsername,omitempty"` + OrganizationID string `json:"organizationId"` + OrganizationName string `json:"organizationName"` + Capability string `json:"capability"` + CapabilityLabel string `json:"capabilityLabel"` + Provider string `json:"provider,omitempty"` + ProviderLabel string `json:"providerLabel"` + ReqKey string `json:"reqKey,omitempty"` + CreatedAt string `json:"createdAt"` +} +type TrendPoint struct { + Date string `json:"date"` + Label string `json:"label"` + Count int `json:"count"` +} +type Summary struct { + Total int `json:"total"` + ActiveAccounts int `json:"activeAccounts"` + ActiveOrganizations int `json:"activeOrganizations"` + AveragePerDay float64 `json:"averagePerDay"` +} +type organizationRow struct { + OrganizationID string `json:"organizationId"` + OrganizationName string `json:"organizationName"` + Count int `json:"count"` + AccountCount int `json:"accountCount"` + LastUsedAt string `json:"lastUsedAt,omitempty"` + accounts map[string]bool +} +type accountRow struct { + OwnerID string `json:"ownerId"` + AccountName string `json:"accountName"` + AccountUsername string `json:"accountUsername,omitempty"` + OrganizationID string `json:"organizationId"` + OrganizationName string `json:"organizationName"` + Count int `json:"count"` + LastUsedAt string `json:"lastUsedAt,omitempty"` +} +type Options struct { + Organizations []Option `json:"organizations"` + Accounts []Option `json:"accounts"` + Capabilities []Option `json:"capabilities"` + Providers []Option `json:"providers"` +} +type Option struct { + Value string `json:"value"` + Label string `json:"label"` +} +type PersonalReport struct { + Preset Preset `json:"preset"` + Range DateRange `json:"range"` + Total int `json:"total"` + ByCapability []CountItem `json:"byCapability"` + Recent []Record `json:"recent"` +} +type AdminReport struct { + Range DateRange `json:"range"` + Summary Summary `json:"summary"` + Trend []TrendPoint `json:"trend"` + ByCapability []CountItem `json:"byCapability"` + ByProvider []CountItem `json:"byProvider"` + Organizations any `json:"organizations"` + Accounts any `json:"accounts"` + Recent any `json:"recent"` + Options Options `json:"options"` +} +type Reporter interface { + Personal(context.Context, PersonalRequest) (PersonalReport, error) + Admin(context.Context, AdminRequest) (AdminReport, error) +} func (s Service) Record(event Event) (*Event, error) { if event.Source == "api" || event.Provider == "mock" { diff --git a/backend/internal/webhook/destination.go b/backend/internal/webhook/destination.go new file mode 100644 index 0000000..069e666 --- /dev/null +++ b/backend/internal/webhook/destination.go @@ -0,0 +1,139 @@ +package webhook + +import ( + "context" + "errors" + "net" + "net/netip" + "net/url" + "strings" +) + +// IPResolver is the DNS seam used by PublicDestinationPolicy. +type IPResolver interface { + LookupIPAddr(context.Context, string) ([]net.IPAddr, error) +} + +// ContextDialer is the network seam used after DNS answers have been checked. +type ContextDialer interface { + DialContext(context.Context, string, string) (net.Conn, error) +} + +// DestinationPolicy validates an outbound HTTP destination. +type DestinationPolicy interface { + Validate(context.Context, *url.URL) error +} + +// PublicDestinationValidator allows only HTTP(S) destinations whose complete DNS +// answer set contains public unicast addresses. DialContext resolves and dials +// one of those checked addresses directly, avoiding a second system DNS lookup. +type PublicDestinationValidator struct { + resolver IPResolver + dialer ContextDialer +} + +func NewPublicDestinationPolicy(resolver IPResolver, dialer ContextDialer) *PublicDestinationValidator { + if resolver == nil { + resolver = net.DefaultResolver + } + if dialer == nil { + dialer = &net.Dialer{} + } + return &PublicDestinationValidator{resolver: resolver, dialer: dialer} +} + +// PublicDestinationPolicy is the compatibility validator factory used by +// callers that provide their own bounded http.Client. +func PublicDestinationPolicy(resolver IPResolver) func(context.Context, *url.URL) error { + return NewPublicDestinationPolicy(resolver, nil).Validate +} + +func (policy *PublicDestinationValidator) Validate(ctx context.Context, target *url.URL) error { + _, err := policy.resolve(ctx, target) + return err +} + +func (policy *PublicDestinationValidator) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, errors.New("invalid outbound address") + } + target := &url.URL{Scheme: "http", Host: net.JoinHostPort(host, port)} + ips, err := policy.resolve(ctx, target) + if err != nil { + return nil, err + } + return policy.dialer.DialContext(ctx, network, net.JoinHostPort(ips[0].String(), port)) +} + +func (policy *PublicDestinationValidator) resolve(ctx context.Context, target *url.URL) ([]net.IP, error) { + if target == nil || (target.Scheme != "http" && target.Scheme != "https") || target.Host == "" || target.User != nil { + return nil, errors.New("outbound destination is invalid") + } + host := strings.TrimSuffix(strings.ToLower(target.Hostname()), ".") + if host == "" || host == "localhost" || strings.HasSuffix(host, ".localhost") { + return nil, errors.New("outbound destination is not public") + } + var ips []net.IP + if literal := net.ParseIP(host); literal != nil { + ips = []net.IP{literal} + } else { + answers, err := policy.resolver.LookupIPAddr(ctx, host) + if err != nil || len(answers) == 0 { + return nil, errors.New("outbound destination DNS lookup failed") + } + for _, answer := range answers { + ips = append(ips, answer.IP) + } + } + for _, ip := range ips { + if !isPublicUnicastIP(ip) { + return nil, errors.New("outbound destination is not public") + } + } + return ips, nil +} + +func isPublicUnicastIP(ip net.IP) bool { + if ip == nil || !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() { + return false + } + address, ok := netip.AddrFromSlice(ip) + if !ok { + return false + } + address = address.Unmap() + for _, prefix := range nonPublicSpecialUsePrefixes { + if prefix.Contains(address) { + return false + } + } + return true +} + +// IsGlobalUnicast deliberately includes several IANA special-purpose ranges. +// These networks are not valid public callback destinations even though the Go +// standard library does not classify all of them as private or link-local. +var nonPublicSpecialUsePrefixes = []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/8"), + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("192.0.0.0/29"), + netip.MustParsePrefix("192.0.0.170/31"), + netip.MustParsePrefix("192.0.2.0/24"), + netip.MustParsePrefix("192.88.99.0/24"), + netip.MustParsePrefix("198.18.0.0/15"), + netip.MustParsePrefix("198.51.100.0/24"), + netip.MustParsePrefix("203.0.113.0/24"), + netip.MustParsePrefix("240.0.0.0/4"), + netip.MustParsePrefix("64:ff9b::/96"), + netip.MustParsePrefix("64:ff9b:1::/48"), + netip.MustParsePrefix("100::/64"), + netip.MustParsePrefix("2001::/32"), + netip.MustParsePrefix("2001:2::/48"), + netip.MustParsePrefix("2001:10::/28"), + netip.MustParsePrefix("2001:20::/28"), + netip.MustParsePrefix("2001:db8::/32"), + netip.MustParsePrefix("2002::/16"), + netip.MustParsePrefix("3fff::/20"), + netip.MustParsePrefix("5f00::/16"), +} diff --git a/backend/internal/webhook/http_sender.go b/backend/internal/webhook/http_sender.go new file mode 100644 index 0000000..53b0528 --- /dev/null +++ b/backend/internal/webhook/http_sender.go @@ -0,0 +1,82 @@ +// Package webhook owns the generation callback wire and transport policy. +package webhook + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/url" + "time" +) + +// NewPublicHTTPSender constructs the production sender with a transport that +// resolves, validates, and then directly dials the same public IP answer. +func NewPublicHTTPSender(timeout time.Duration, policy *PublicDestinationValidator) (*HTTPSender, error) { + if policy == nil || timeout <= 0 { + return nil, errors.New("webhook HTTP sender requires a bounded client and destination policy") + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.DialContext = policy.DialContext + return NewHTTPSender(&http.Client{Timeout: timeout, Transport: transport}, policy.Validate) +} + +// HTTPSender is a bounded callback transport. Destination policy remains +// injectable at composition time so deployments can enforce DNS/IP controls +// without coupling the byte/HMAC contract to one network environment. +type HTTPSender struct { + client *http.Client + validate func(context.Context, *url.URL) error +} + +func NewHTTPSender(client *http.Client, validate func(context.Context, *url.URL) error) (*HTTPSender, error) { + if client == nil || client.Timeout <= 0 || validate == nil { + return nil, errors.New("webhook HTTP sender requires a bounded client and destination policy") + } + copy := *client + previous := copy.CheckRedirect + copy.CheckRedirect = func(request *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return errors.New("webhook redirect limit exceeded") + } + if err := validate(request.Context(), request.URL); err != nil { + return err + } + if previous != nil { + return previous(request, via) + } + return nil + } + return &HTTPSender{client: ©, validate: validate}, nil +} + +func (sender *HTTPSender) Send(ctx context.Context, input Request) (Response, error) { + target, err := url.Parse(input.URL) + if err != nil || (target.Scheme != "http" && target.Scheme != "https") || target.Host == "" || target.User != nil { + return Response{}, errors.New("webhook destination is invalid") + } + if err := sender.validate(ctx, target); err != nil { + return Response{}, errors.New("webhook destination is not allowed") + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, target.String(), bytes.NewReader(input.Body)) + if err != nil { + return Response{}, errors.New("build webhook request") + } + for key, value := range input.Headers { + request.Header.Set(key, value) + } + response, err := sender.client.Do(request) + if err != nil { + if ctx.Err() != nil { + return Response{}, ctx.Err() + } + return Response{}, errors.New("send webhook request") + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 64<<10)) + return Response{Status: response.StatusCode}, nil +} + +var _ Sender = (*HTTPSender)(nil) diff --git a/backend/internal/webhook/http_sender_test.go b/backend/internal/webhook/http_sender_test.go new file mode 100644 index 0000000..9c98ef6 --- /dev/null +++ b/backend/internal/webhook/http_sender_test.go @@ -0,0 +1,197 @@ +package webhook + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "net/url" + "strings" + "testing" + "time" +) + +type lookupIPFunc func(context.Context, string) ([]net.IPAddr, error) + +func (function lookupIPFunc) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) { + return function(ctx, host) +} + +type contextDialFunc func(context.Context, string, string) (net.Conn, error) + +func (function contextDialFunc) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + return function(ctx, network, address) +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (function roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return function(request) +} + +func TestHTTPSenderPreservesSignedRequestAndReturnsStatus(t *testing.T) { + client := &http.Client{Timeout: time.Second, Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + body, _ := io.ReadAll(request.Body) + if request.Method != http.MethodPost || request.URL.String() != "https://hooks.example.test/job" || string(body) != `{"jobId":"job"}` || request.Header.Get("X-Zhinian-Signature") != "sha256=abc" { + t.Fatalf("request=%s %s body=%s headers=%v", request.Method, request.URL, body, request.Header) + } + return &http.Response{StatusCode: 204, Body: io.NopCloser(strings.NewReader("ignored")), Header: make(http.Header)}, nil + })} + sender, err := NewHTTPSender(client, func(context.Context, *url.URL) error { return nil }) + if err != nil { + t.Fatal(err) + } + response, err := sender.Send(context.Background(), Request{URL: "https://hooks.example.test/job", Body: []byte(`{"jobId":"job"}`), Headers: map[string]string{"X-Zhinian-Signature": "sha256=abc"}}) + if err != nil || response.Status != 204 { + t.Fatalf("Send=%#v,%v", response, err) + } +} + +func TestHTTPSenderRejectsInvalidAndPolicyDeniedDestinations(t *testing.T) { + client := &http.Client{Timeout: time.Second, Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + t.Fatal("transport must not run") + return nil, nil + })} + sender, _ := NewHTTPSender(client, func(context.Context, *url.URL) error { return errors.New("private address") }) + for _, target := range []string{"file:///etc/passwd", "https://user:secret@example.test", "https://127.0.0.1/hook"} { + if _, err := sender.Send(context.Background(), Request{URL: target}); err == nil || strings.Contains(err.Error(), "secret") { + t.Fatalf("target=%q error=%v", target, err) + } + } +} + +func TestHTTPSenderRevalidatesEveryRedirect(t *testing.T) { + validated := make([]string, 0, 2) + client := &http.Client{Timeout: time.Second, Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.URL.Hostname() != "public.test" { + t.Fatal("redirect target transport must not run") + } + return &http.Response{StatusCode: http.StatusTemporaryRedirect, Header: http.Header{"Location": []string{"https://private.test/hook"}}, Body: io.NopCloser(strings.NewReader("")), Request: request}, nil + })} + sender, err := NewHTTPSender(client, func(_ context.Context, target *url.URL) error { + validated = append(validated, target.Hostname()) + if target.Hostname() == "private.test" { + return errors.New("private destination") + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if _, err := sender.Send(context.Background(), Request{URL: "https://public.test/hook"}); err == nil { + t.Fatal("policy-denied redirect was accepted") + } + if strings.Join(validated, ",") != "public.test,private.test" { + t.Fatalf("validated destinations = %v", validated) + } +} + +func TestNewHTTPSenderRequiresBoundedClientAndPolicy(t *testing.T) { + if _, err := NewHTTPSender(nil, func(context.Context, *url.URL) error { return nil }); err == nil { + t.Fatal("nil client accepted") + } + if _, err := NewHTTPSender(&http.Client{}, func(context.Context, *url.URL) error { return nil }); err == nil { + t.Fatal("unbounded client accepted") + } + if _, err := NewHTTPSender(&http.Client{Timeout: time.Second}, nil); err == nil { + t.Fatal("nil policy accepted") + } +} + +func TestPublicDestinationPolicyRejectsNonPublicDestinations(t *testing.T) { + tests := []struct { + name string + url string + ips []net.IPAddr + }{ + {name: "userinfo", url: "https://user:secret@example.test/hook", ips: []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}}, + {name: "localhost name", url: "https://localhost/hook", ips: []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}}, + {name: "loopback", url: "https://example.test/hook", ips: []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}}}, + {name: "private", url: "https://example.test/hook", ips: []net.IPAddr{{IP: net.ParseIP("10.0.0.1")}}}, + {name: "link local", url: "https://example.test/hook", ips: []net.IPAddr{{IP: net.ParseIP("169.254.169.254")}}}, + {name: "multicast", url: "https://example.test/hook", ips: []net.IPAddr{{IP: net.ParseIP("224.0.0.1")}}}, + {name: "unspecified", url: "https://example.test/hook", ips: []net.IPAddr{{IP: net.ParseIP("0.0.0.0")}}}, + {name: "mixed answers", url: "https://example.test/hook", ips: []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}, {IP: net.ParseIP("192.168.1.1")}}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + policy := NewPublicDestinationPolicy(lookupIPFunc(func(context.Context, string) ([]net.IPAddr, error) { return test.ips, nil }), nil) + u, _ := url.Parse(test.url) + if err := policy.Validate(context.Background(), u); err == nil || strings.Contains(err.Error(), "secret") { + t.Fatalf("Validate(%q) = %v", test.url, err) + } + }) + } +} + +func TestPublicDestinationPolicyRejectsSpecialUseUnicastAddresses(t *testing.T) { + addresses := []string{ + "100.64.0.1", // shared address space (CGNAT) + "192.0.0.1", // IETF protocol assignments + "192.0.2.1", // documentation + "198.18.0.1", // benchmarking + "198.51.100.1", // documentation + "203.0.113.1", // documentation + "240.0.0.1", // reserved + "64:ff9b::a00:1", // IPv4/IPv6 translation can embed private IPv4 + "100::1", // discard-only + "2001::1", // Teredo can embed non-public IPv4 + "2001:2::1", // benchmarking + "2001:db8::1", // documentation + "2001:10::1", // deprecated ORCHID + "2001:20::1", // ORCHIDv2 + } + for _, address := range addresses { + t.Run(address, func(t *testing.T) { + policy := NewPublicDestinationPolicy(nil, nil) + target, err := url.Parse("https://[" + address + "]/hook") + if net.ParseIP(address).To4() != nil { + target, err = url.Parse("https://" + address + "/hook") + } + if err != nil { + t.Fatal(err) + } + if err := policy.Validate(context.Background(), target); err == nil { + t.Fatalf("special-use address %s was accepted", address) + } + }) + } +} + +func TestPublicDestinationPolicyFailsClosedOnDNSFailure(t *testing.T) { + policy := NewPublicDestinationPolicy(lookupIPFunc(func(context.Context, string) ([]net.IPAddr, error) { + return nil, errors.New("resolver unavailable") + }), nil) + u, _ := url.Parse("https://hooks.example.test/job") + if err := policy.Validate(context.Background(), u); err == nil { + t.Fatal("DNS failure was accepted") + } +} + +func TestPublicDestinationPolicyAllowsPublicAddress(t *testing.T) { + policy := NewPublicDestinationPolicy(lookupIPFunc(func(context.Context, string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil + }), nil) + u, _ := url.Parse("https://hooks.example.test/job") + if err := policy.Validate(context.Background(), u); err != nil { + t.Fatalf("public destination rejected: %v", err) + } +} + +func TestPublicDestinationPolicyDialsValidatedIPAddress(t *testing.T) { + var dialed string + policy := NewPublicDestinationPolicy( + lookupIPFunc(func(context.Context, string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil + }), + contextDialFunc(func(_ context.Context, _, address string) (net.Conn, error) { + dialed = address + return nil, errors.New("dial stopped by test") + }), + ) + _, _ = policy.DialContext(context.Background(), "tcp", "hooks.example.test:443") + if dialed != "93.184.216.34:443" { + t.Fatalf("dialed address = %q", dialed) + } +} diff --git a/components/settings-panel.tsx b/components/settings-panel.tsx index bb872e6..1b13b53 100644 --- a/components/settings-panel.tsx +++ b/components/settings-panel.tsx @@ -23,6 +23,7 @@ type SettingsGroup = { }; type SettingsPayload = { + restartRequired?: boolean; services: { visual: boolean; evolink: boolean; @@ -121,7 +122,7 @@ export function SettingsPanel() { if (!response.ok) throw new Error(nextPayload.error || "保存设置失败"); setPayload(nextPayload); setValues(valuesFromPayload(nextPayload)); - setMessage("配置已保存并应用到当前服务。"); + setMessage(nextPayload.restartRequired ? "配置已保存,重启 Go 服务后生效。" : "配置已保存并应用到当前服务。"); } catch (err) { setError(err instanceof Error ? err.message : String(err)); } finally { diff --git a/contracts/admin/http-auth-compat-v1.json b/contracts/admin/http-auth-compat-v1.json new file mode 100644 index 0000000..233befb --- /dev/null +++ b/contracts/admin/http-auth-compat-v1.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "routes": [ + { "method": "GET", "path": "/api/auth/login", "status": 307, "location": "/auth/login" }, + { "method": "GET", "path": "/api/auth/callback", "status": 307, "location": "/auth/login?error=callback_failed" }, + { "method": "GET", "path": "/api/auth/captcha", "status": 200, "body": { "enabled": false, "message": "平台账号登录不使用外部验证码。" } } + ] +} diff --git a/contracts/admin/http-v1.json b/contracts/admin/http-v1.json new file mode 100644 index 0000000..1fdaabe --- /dev/null +++ b/contracts/admin/http-v1.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "routes": [ + { "path": "/api/admin/accounts", "methods": ["GET", "POST", "PATCH", "PUT", "DELETE"], "requirement": "admin" }, + { "path": "/api/admin/accounts/password", "methods": ["POST"], "requirement": "admin" }, + { "path": "/api/admin/organizations", "methods": ["GET", "POST", "PATCH", "DELETE"], "requirement": "admin" }, + { "path": "/api/admin/accounts/groups", "methods": ["POST"], "requirement": "public-compatibility", "status": 410 } + ], + "accountProjection": ["id", "phone", "displayName", "role", "organizationId", "status", "createdAt", "lastLoginAt", "lockedUntil"], + "organizationProjection": ["id", "name", "status"], + "secretFields": ["passwordHash", "passwordSalt", "failedLoginCount", "sessionVersion", "legacySubject"] +} diff --git a/contracts/assets/http-v1.json b/contracts/assets/http-v1.json new file mode 100644 index 0000000..5cad396 --- /dev/null +++ b/contracts/assets/http-v1.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "limits": { "jsonBytes": 1048576, "uploadBytes": 20971520, "remoteBytes": 20971520 }, + "errors": { + "platformNotFound": { "status": 404, "body": { "error": "资产不存在" } }, + "platformNotDownloadable": { "status": 404, "body": { "error": "资产文件不可下载" } }, + "publicNotFound": { "status": 404, "body": { "error": "Asset not found." } }, + "publicNotDownloadable": { "status": 404, "body": { "error": "Asset file is not downloadable." } }, + "noFiles": { "status": 400, "body": { "error": "No files uploaded." } }, + "tooLarge": { "status": 413, "body": { "error": "Request body is too large." } }, + "internal": { "status": 500, "body": { "error": "Internal server error." } } + }, + "methods": { + "headExecutesGet": true, + "optionsStatus": 204, + "unsupportedStatus": 405, + "unsupportedBody": "empty" + }, + "binary": { + "downloadCacheControl": "private, no-store", + "servedCacheControl": "public, max-age=31536000, immutable" + }, + "routes": [ + { "path": "/api/assets", "allow": "GET, HEAD, POST, OPTIONS" }, + { "path": "/api/assets/upload", "allow": "POST, OPTIONS" }, + { "path": "/api/assets/{id}", "allow": "DELETE, OPTIONS" }, + { "path": "/api/assets/{id}/download", "allow": "GET, HEAD, OPTIONS" }, + { "path": "/api/v1/assets", "allow": "GET, HEAD, POST, OPTIONS" }, + { "path": "/api/v1/assets/{id}", "allow": "GET, HEAD, OPTIONS" }, + { "path": "/api/v1/assets/{id}/download", "allow": "GET, HEAD, OPTIONS" }, + { "path": "/uploads/{path...}", "allow": "GET, HEAD, OPTIONS" }, + { "path": "/generated-results/{path...}", "allow": "GET, HEAD, OPTIONS" } + ] +} diff --git a/contracts/auth/password-change-v1.json b/contracts/auth/password-change-v1.json new file mode 100644 index 0000000..bf982c0 --- /dev/null +++ b/contracts/auth/password-change-v1.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "method": "POST", + "path": "/api/auth/password/change", + "request": ["currentPassword", "newPassword", "confirmPassword"], + "minimumPasswordLength": 8, + "successStatus": 200, + "replacementCookie": "zhinian_session", + "incrementsSessionVersion": true, + "publicUserExcludes": ["passwordHash", "passwordSalt", "sessionVersion", "accessToken", "tokenType"] +} diff --git a/contracts/billing/http-v1.json b/contracts/billing/http-v1.json new file mode 100644 index 0000000..c8c359e --- /dev/null +++ b/contracts/billing/http-v1.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "currency": "CNY", + "moneyUnit": "fen", + "routes": [ + { "method": "GET", "path": "/api/billing", "requirement": "app", "organizationSource": "refreshed_session", "unboundStatus": 422 }, + { "method": "POST", "path": "/api/billing/quote", "requirement": "app", "accountSource": "refreshed_session" }, + { "method": "GET", "path": "/api/admin/billing", "requirement": "super_admin" }, + { "method": "PATCH", "path": "/api/admin/billing/account", "requirement": "super_admin", "configuration": "injected_store" }, + { "method": "POST", "path": "/api/admin/billing/adjustments", "requirement": "super_admin" }, + { "method": "GET", "path": "/api/admin/billing/prices", "requirement": "super_admin" }, + { "method": "PATCH", "path": "/api/admin/billing/prices/{id}", "requirement": "super_admin", "allowedFields": ["markupMultiplier", "dimensionKey", "tierValue"] } + ], + "statuses": { "insufficientBalance": 402, "idempotencyConflict": 409, "unboundOrganization": 422, "infrastructure": 500 }, + "pricePatch": { "minimum": 1, "maximum": 1000, "precision": 4, "tierPairRequired": true } +} diff --git a/contracts/jobs/http-v1.json b/contracts/jobs/http-v1.json new file mode 100644 index 0000000..e3e9707 --- /dev/null +++ b/contracts/jobs/http-v1.json @@ -0,0 +1,9 @@ +{ + "version": 1, + "platform": { + "image": {"collection": "/api/generations/image", "methods": ["GET", "POST"], "item": "/api/generations/image/{id}", "itemMethods": ["GET", "DELETE"], "retry": "/api/generations/image/{id}/retry", "retryMethods": ["POST"]}, + "video": {"collection": "/api/generations/video", "methods": ["GET", "POST"], "item": "/api/generations/video/{id}", "itemMethods": ["GET", "DELETE"]} + }, + "public": {"collection": "/api/v1/jobs", "methods": ["GET", "POST"], "item": "/api/v1/jobs/{id}", "itemMethods": ["GET"], "cancel": "/api/v1/jobs/{id}/cancel", "cancelMethods": ["POST"], "scope": ["ownerId", "externalClientId"], "statuses": {"created": 202, "reused": 200, "conflict": 409, "notFound": 404}}, + "worker": {"path": "/api/internal/worker/tick", "methods": ["POST"], "auth": "publicapi.AssertInternalWorker"} +} diff --git a/contracts/logs/runtime-v1.json b/contracts/logs/runtime-v1.json new file mode 100644 index 0000000..32d9431 --- /dev/null +++ b/contracts/logs/runtime-v1.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "fileName": "server-events.jsonl", + "rotatedSuffix": ".1", + "defaultLimit": 100, + "minimumLimit": 1, + "maximumLimit": 500, + "maximumTextLength": 20000, + "maximumDepth": 5, + "maximumArrayItems": 50, + "maximumObjectKeys": 80, + "corruptLines": "skip", + "order": "newest-first", + "redactedValue": "[redacted]", + "redactedJwt": "[jwt-redacted]" +} diff --git a/contracts/providers/http-v1.json b/contracts/providers/http-v1.json new file mode 100644 index 0000000..3c4cc7f --- /dev/null +++ b/contracts/providers/http-v1.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "providers": { + "volcengine-visual": {"submitAction": "CVSync2AsyncSubmitTask", "queryAction": "CVSync2AsyncGetResult", "auth": "HMAC-SHA256"}, + "evolink": {"submit": "/v1/images/generations", "query": "/v1/tasks/{id}", "auth": "bearer"}, + "bailian": {"imageSubmit": "/api/v1/services/aigc/image-generation/generation", "videoSubmit": "/api/v1/services/aigc/video-generation/video-synthesis", "query": "/api/v1/tasks/{id}", "auth": "bearer"}, + "seedance": {"submit": "/contents/generations/tasks", "query": "/contents/generations/tasks/{id}", "auth": "bearer"}, + "mock": {"deterministic": true} + }, + "errors": {"generic": true, "secretSafe": true}, + "liveCallsInTests": false +} diff --git a/contracts/settings/runtime-v1.json b/contracts/settings/runtime-v1.json new file mode 100644 index 0000000..b826c0c --- /dev/null +++ b/contracts/settings/runtime-v1.json @@ -0,0 +1,45 @@ +{ + "version": 1, + "fileName": ".env.local", + "allowedKeys": [ + "ZHINIAN_AUTH_REQUIRED", + "ZHINIAN_AUTH_SESSION_SECRET", + "ZHINIAN_BILLING_REQUIRED", + "ZHINIAN_BILLING_ACCOUNT_NAME", + "ZHINIAN_BILLING_ACCOUNT_BANK", + "ZHINIAN_BILLING_ACCOUNT_NUMBER", + "ZHINIAN_BILLING_CONTACT", + "VOLCENGINE_ACCESS_KEY_ID", + "VOLCENGINE_SECRET_ACCESS_KEY", + "EVOLINK_API_KEY", + "EVOLINK_BASE_URL", + "EVOLINK_IMAGE_MODEL", + "EVOLINK_IMAGE_QUALITY", + "SEEDANCE_API_KEY", + "BAILIAN_API_KEY", + "BAILIAN_BASE_URL", + "BAILIAN_IMAGE_MODEL", + "BAILIAN_VIDEO_MODEL", + "ALI_OSS_ENDPOINT", + "ALI_OSS_BUCKET", + "ALI_OSS_ACCESS_KEY_ID", + "ALI_OSS_ACCESS_KEY_SECRET", + "ALI_OSS_PUBLIC_BASE_URL", + "IMAGE_GENERATE_ENGINE", + "VIDEO_GENERATE_ENGINE" + ], + "secretKeys": [ + "ZHINIAN_AUTH_SESSION_SECRET", + "VOLCENGINE_ACCESS_KEY_ID", + "VOLCENGINE_SECRET_ACCESS_KEY", + "EVOLINK_API_KEY", + "SEEDANCE_API_KEY", + "BAILIAN_API_KEY", + "ALI_OSS_ACCESS_KEY_ID", + "ALI_OSS_ACCESS_KEY_SECRET" + ], + "emptySecretInput": "preserve", + "emptyNonSecretInput": "write-empty", + "publicSecretValue": "blank", + "runtimeApplication": "restart-required" +} diff --git a/contracts/usage/http-v1.json b/contracts/usage/http-v1.json new file mode 100644 index 0000000..6266a0e --- /dev/null +++ b/contracts/usage/http-v1.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "routes": [ + { "method": "GET", "path": "/api/usage", "requirement": "app", "accountSource": "refreshed_session", "defaultPreset": "month" }, + { "method": "GET", "path": "/api/admin/usage", "requirement": "admin", "organizationAdminScope": "refreshed_session", "organizationAdminRedactions": ["accounts", "recent", "options.accounts"] } + ], + "presets": ["today", "7d", "30d", "month"], + "capabilities": ["image.generate", "video.generate"], + "providers": ["volcengine-visual", "evolink", "seedance", "bailian"], + "infrastructureStatus": 500 +} diff --git a/database/migrations/0002_generation_lifecycle_fencing.sql b/database/migrations/0002_generation_lifecycle_fencing.sql new file mode 100644 index 0000000..22481b7 --- /dev/null +++ b/database/migrations/0002_generation_lifecycle_fencing.sql @@ -0,0 +1,69 @@ +-- Recoverable generation finalization and charge-before-dispatch fencing. +-- Additive so the previous release remains rollback-compatible. + +alter table public.generation_jobs + add column if not exists dispatch_ready_at timestamptz; + +alter table public.generation_jobs + add column if not exists finalized_at timestamptz; + +alter table public.generation_jobs + add column if not exists provider_dispatch_started_at timestamptz; + +-- Existing rows were created before dispatch fencing and are therefore ready. +update public.generation_jobs +set dispatch_ready_at = coalesce(dispatch_ready_at, created_at, now()) +where dispatch_ready_at is null; + +-- Rows already terminal before this migration have completed their legacy +-- finalization path and must not be replayed into billing/webhook side effects. +update public.generation_jobs +set finalized_at = coalesce(finalized_at, completed_at, updated_at, now()) +where finalized_at is null + and status in ('succeeded', 'failed', 'expired', 'cancelled'); + +create index if not exists generation_jobs_finalize_claim_idx + on public.generation_jobs(finalized_at, scheduled_at, locked_at, priority desc); + +create or replace function public.claim_generation_jobs( + p_worker_id text, + p_limit integer default 1, + p_lock_timeout_seconds integer default 300 +) +returns setof public.generation_jobs +language plpgsql +set search_path = public, pg_temp +as $$ +declare + v_now timestamptz := now(); +begin + return query + with candidates as ( + select id + from public.generation_jobs + where dispatch_ready_at is not null + and finalized_at is null + and status in ('queued', 'running', 'succeeded', 'failed', 'expired', 'cancelled') + and coalesce(scheduled_at, created_at) <= v_now + and ( + locked_at is null + or locked_at < v_now - make_interval(secs => p_lock_timeout_seconds) + ) + order by coalesce(priority, 0) desc, coalesce(scheduled_at, created_at) asc, created_at asc + limit greatest(1, least(coalesce(p_limit, 1), 20)) + for update skip locked + ), + updated as ( + update public.generation_jobs + set locked_at = v_now, + locked_by = p_worker_id, + started_at = coalesce(public.generation_jobs.started_at, v_now), + updated_at = v_now + where id in (select id from candidates) + returning public.generation_jobs.* + ) + select * from updated; +end; +$$; + +revoke all on function public.claim_generation_jobs(text, integer, integer) from public; diff --git a/supabase/schema.sql b/supabase/schema.sql index ae7bfd4..ca5e2e4 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -29,6 +29,7 @@ create table if not exists generation_jobs ( input_urls text[] not null default '{}', output_asset_ids text[] not null default '{}', provider_task_id text, + provider_dispatch_started_at timestamptz, request_payload jsonb not null default '{}'::jsonb, response_payload jsonb, error jsonb, @@ -43,6 +44,8 @@ create table if not exists generation_jobs ( locked_by text, started_at timestamptz, completed_at timestamptz, + dispatch_ready_at timestamptz, + finalized_at timestamptz, webhook_url text, webhook_attempts integer not null default 0, webhook_last_status jsonb, @@ -66,6 +69,18 @@ alter table generation_jobs add column if not exists webhook_url text; alter table generation_jobs add column if not exists webhook_attempts integer not null default 0; alter table generation_jobs add column if not exists webhook_last_status jsonb; alter table generation_jobs add column if not exists usage_context jsonb; +alter table generation_jobs add column if not exists provider_dispatch_started_at timestamptz; +alter table generation_jobs add column if not exists dispatch_ready_at timestamptz; +alter table generation_jobs add column if not exists finalized_at timestamptz; + +update generation_jobs +set dispatch_ready_at = coalesce(dispatch_ready_at, created_at, now()) +where dispatch_ready_at is null; + +update generation_jobs +set finalized_at = coalesce(finalized_at, completed_at, updated_at, now()) +where finalized_at is null + and status in ('succeeded', 'failed', 'expired', 'cancelled'); create table if not exists usage_events ( id text primary key, @@ -147,6 +162,7 @@ create index if not exists assets_owner_created_idx on assets(owner_id, created_ create index if not exists generation_jobs_owner_created_idx on generation_jobs(owner_id, created_at desc); create index if not exists generation_jobs_status_idx on generation_jobs(status); create index if not exists generation_jobs_claim_idx on generation_jobs(status, scheduled_at, locked_at, priority desc); +create index if not exists generation_jobs_finalize_claim_idx on generation_jobs(finalized_at, scheduled_at, locked_at, priority desc); create index if not exists generation_jobs_external_client_idx on generation_jobs(owner_id, external_client_id, created_at desc); create unique index if not exists generation_jobs_idempotency_idx on generation_jobs(owner_id, external_client_id, idempotency_key) @@ -187,7 +203,9 @@ begin with candidates as ( select id from generation_jobs - where status in ('queued', 'running') + where dispatch_ready_at is not null + and finalized_at is null + and status in ('queued', 'running', 'succeeded', 'failed', 'expired', 'cancelled') and coalesce(scheduled_at, created_at) <= v_now and ( locked_at is null diff --git a/tests/admin-http-auth-compat.test.ts b/tests/admin-http-auth-compat.test.ts new file mode 100644 index 0000000..e6f876c --- /dev/null +++ b/tests/admin-http-auth-compat.test.ts @@ -0,0 +1,12 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +describe("Go legacy auth compatibility HTTP v1 contract", () => { + it("pins login, callback, and disabled captcha endpoints", async () => { + const fixture = JSON.parse(await readFile(new URL("../contracts/admin/http-auth-compat-v1.json", import.meta.url), "utf8")); + const source = await readFile(new URL("../backend/internal/httpapi/auth_compat.go", import.meta.url), "utf8"); + expect(fixture.routes.map((route: { path: string }) => route.path)).toEqual(["/api/auth/login", "/api/auth/callback", "/api/auth/captcha"]); + expect(source).toContain("callback_failed"); + expect(source).toContain("平台账号登录不使用外部验证码。"); + }); +}); diff --git a/tests/admin-http-contract.test.ts b/tests/admin-http-contract.test.ts new file mode 100644 index 0000000..5517d94 --- /dev/null +++ b/tests/admin-http-contract.test.ts @@ -0,0 +1,17 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +describe("Go administration HTTP v1 contract", () => { + it("pins methods, authorization, and security projections", async () => { + const fixture = JSON.parse(await readFile(new URL("../contracts/admin/http-v1.json", import.meta.url), "utf8")); + const source = await readFile(new URL("../backend/internal/httpapi/admin.go", import.meta.url), "utf8"); + expect(fixture.routes).toHaveLength(4); + expect(fixture.routes.filter((route: { path: string }) => route.path !== "/api/admin/accounts/groups").every((route: { requirement: string }) => route.requirement === "admin")).toBe(true); + expect(fixture.routes.find((route: { path: string }) => route.path === "/api/admin/accounts/groups")?.requirement).toBe("public-compatibility"); + expect(source).toContain("PlatformAdmin"); + expect(source).toContain("http.StatusGone"); + const projection = source.slice(source.indexOf("func adminAccountProjection(")); + for (const field of fixture.secretFields) expect(projection).not.toContain(`\"${field}\"`); + for (const field of fixture.accountProjection) expect(source).toContain(`\"${field}\"`); + }); +}); diff --git a/tests/assets-http-contract.test.ts b/tests/assets-http-contract.test.ts new file mode 100644 index 0000000..398f2d1 --- /dev/null +++ b/tests/assets-http-contract.test.ts @@ -0,0 +1,21 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +describe("assets HTTP v1 compatibility contract", () => { + it("freezes route, limit, safe-error, and binary-cache semantics", async () => { + const contract = JSON.parse(await readFile(new URL("../contracts/assets/http-v1.json", import.meta.url), "utf8")); + expect(contract.version).toBe(1); + expect(contract.limits).toEqual({ jsonBytes: 1_048_576, uploadBytes: 20_971_520, remoteBytes: 20_971_520 }); + expect(contract.routes.map((route: { path: string }) => route.path)).toEqual([ + "/api/assets", "/api/assets/upload", "/api/assets/{id}", "/api/assets/{id}/download", + "/api/v1/assets", "/api/v1/assets/{id}", "/api/v1/assets/{id}/download", + "/uploads/{path...}", "/generated-results/{path...}" + ]); + expect(contract.errors.internal.body).toEqual({ error: "Internal server error." }); + expect(contract.methods).toEqual({ headExecutesGet: true, optionsStatus: 204, unsupportedStatus: 405, unsupportedBody: "empty" }); + expect(contract.binary).toEqual({ + downloadCacheControl: "private, no-store", + servedCacheControl: "public, max-age=31536000, immutable" + }); + }); +}); diff --git a/tests/auth-password-change-contract.test.ts b/tests/auth-password-change-contract.test.ts new file mode 100644 index 0000000..22165b0 --- /dev/null +++ b/tests/auth-password-change-contract.test.ts @@ -0,0 +1,17 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +describe("Go password change v1 contract", () => { + it("pins the transactional password and replacement-cookie implementation", async () => { + const fixture = JSON.parse(await readFile(new URL("../contracts/auth/password-change-v1.json", import.meta.url), "utf8")); + const http = await readFile(new URL("../backend/internal/httpapi/auth_password_change.go", import.meta.url), "utf8"); + const identity = await readFile(new URL("../backend/internal/identity/password_change.go", import.meta.url), "utf8"); + const postgres = await readFile(new URL("../backend/internal/postgres/password_change.go", import.meta.url), "utf8"); + expect(fixture).toMatchObject({ version: 1, path: "/api/auth/password/change", minimumPasswordLength: 8, incrementsSessionVersion: true }); + expect(http).toContain("identity.SetSessionCookies"); + expect(http).toContain("PlatformApp"); + expect(identity).toContain("len(command.NewPassword) < 8"); + expect(postgres).toContain("FOR UPDATE"); + expect(postgres).toContain("session_version = session_version + 1"); + }); +}); diff --git a/tests/billing-http-contract.test.ts b/tests/billing-http-contract.test.ts new file mode 100644 index 0000000..4478bd9 --- /dev/null +++ b/tests/billing-http-contract.test.ts @@ -0,0 +1,17 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +describe("billing HTTP compatibility fixture", () => { + it("freezes routes, refreshed-session scope, money, and admin mutation rules", async () => { + const fixture = JSON.parse(await readFile("contracts/billing/http-v1.json", "utf8")); + expect(fixture).toMatchObject({ + version: 1, currency: "CNY", moneyUnit: "fen", + statuses: { insufficientBalance: 402, idempotencyConflict: 409, unboundOrganization: 422, infrastructure: 500 }, + pricePatch: { minimum: 1, maximum: 1000, precision: 4, tierPairRequired: true } + }); + expect(fixture.routes).toEqual(expect.arrayContaining([ + expect.objectContaining({ method: "GET", path: "/api/billing", organizationSource: "refreshed_session" }), + expect.objectContaining({ method: "PATCH", path: "/api/admin/billing/prices/{id}", requirement: "super_admin" }) + ])); + }); +}); diff --git a/tests/jobs-http-contract.test.ts b/tests/jobs-http-contract.test.ts new file mode 100644 index 0000000..972633d --- /dev/null +++ b/tests/jobs-http-contract.test.ts @@ -0,0 +1,3 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; +describe("Go jobs HTTP v1 contract", () => { it("freezes paths, methods, scope and statuses", async () => { const c=JSON.parse(await readFile(new URL("../contracts/jobs/http-v1.json",import.meta.url),"utf8")); expect(c.public.scope).toEqual(["ownerId","externalClientId"]); expect(c.public.statuses).toEqual({created:202,reused:200,conflict:409,notFound:404}); expect(c.platform.image.methods).toEqual(["GET","POST"]); expect(c.worker.auth).toBe("publicapi.AssertInternalWorker"); }); }); diff --git a/tests/logs-go-contract.test.ts b/tests/logs-go-contract.test.ts new file mode 100644 index 0000000..eb78928 --- /dev/null +++ b/tests/logs-go-contract.test.ts @@ -0,0 +1,26 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +const fixtureUrl = new URL("../contracts/logs/runtime-v1.json", import.meta.url); + +describe("Go runtime logs compatibility fixture", () => { + it("freezes storage, bounds, ordering, and redaction behavior", async () => { + const fixture = JSON.parse(await readFile(fixtureUrl, "utf8")); + expect(fixture).toEqual({ + version: 1, + fileName: "server-events.jsonl", + rotatedSuffix: ".1", + defaultLimit: 100, + minimumLimit: 1, + maximumLimit: 500, + maximumTextLength: 20000, + maximumDepth: 5, + maximumArrayItems: 50, + maximumObjectKeys: 80, + corruptLines: "skip", + order: "newest-first", + redactedValue: "[redacted]", + redactedJwt: "[jwt-redacted]" + }); + }); +}); diff --git a/tests/providers-contract.test.ts b/tests/providers-contract.test.ts new file mode 100644 index 0000000..9f5a61c --- /dev/null +++ b/tests/providers-contract.test.ts @@ -0,0 +1,3 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; +describe("Go provider adapter contract",()=>{it("freezes bounded adapters and safe errors",async()=>{const c=JSON.parse(await readFile(new URL("../contracts/providers/http-v1.json",import.meta.url),"utf8"));expect(Object.keys(c.providers)).toEqual(["volcengine-visual","evolink","bailian","seedance","mock"]);expect(c.errors).toEqual({generic:true,secretSafe:true});expect(c.liveCallsInTests).toBe(false);});}); diff --git a/tests/settings-go-contract.test.ts b/tests/settings-go-contract.test.ts new file mode 100644 index 0000000..c008c4c --- /dev/null +++ b/tests/settings-go-contract.test.ts @@ -0,0 +1,21 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +const fixtureUrl = new URL("../contracts/settings/runtime-v1.json", import.meta.url); + +describe("Go runtime settings compatibility fixture", () => { + it("freezes the editable whitelist and secret projection rules", async () => { + const fixture = JSON.parse(await readFile(fixtureUrl, "utf8")); + expect(fixture).toMatchObject({ + version: 1, + fileName: ".env.local", + emptySecretInput: "preserve", + emptyNonSecretInput: "write-empty", + publicSecretValue: "blank", + runtimeApplication: "restart-required" + }); + expect(fixture.allowedKeys).toContain("IMAGE_GENERATE_ENGINE"); + expect(fixture.secretKeys).toContain("ALI_OSS_ACCESS_KEY_SECRET"); + expect(fixture.allowedKeys).not.toContain("DATABASE_URL"); + }); +}); diff --git a/tests/usage-http-contract.test.ts b/tests/usage-http-contract.test.ts new file mode 100644 index 0000000..582ef5a --- /dev/null +++ b/tests/usage-http-contract.test.ts @@ -0,0 +1,15 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +describe("usage HTTP compatibility fixture", () => { + it("freezes personal/admin scope and filters", async () => { + const fixture = JSON.parse(await readFile("contracts/usage/http-v1.json", "utf8")); + expect(fixture).toMatchObject({ + version: 1, + presets: ["today", "7d", "30d", "month"], + capabilities: ["image.generate", "video.generate"], + infrastructureStatus: 500 + }); + expect(fixture.routes).toContainEqual(expect.objectContaining({ path: "/api/admin/usage", requirement: "admin", organizationAdminScope: "refreshed_session" })); + }); +}); diff --git a/tests/wallet-sql-contract.test.ts b/tests/wallet-sql-contract.test.ts index 343ebb5..dca7f95 100644 --- a/tests/wallet-sql-contract.test.ts +++ b/tests/wallet-sql-contract.test.ts @@ -8,6 +8,8 @@ const schemaPaths = [ "../supabase/schema.sql" ]; +const lifecycleMigrationPath = "../database/migrations/0002_generation_lifecycle_fencing.sql"; + function readSchema(relativePath: string): string { return readFileSync(fileURLToPath(new URL(relativePath, import.meta.url)), "utf8").replace(/\r\n/g, "\n"); } @@ -47,12 +49,29 @@ describe.each(schemaPaths)("wallet SQL contract: %s", (schemaPath) => { }); }); -it("keeps the migration and Supabase compatibility snapshot synchronized", () => { - const migration = readSchema(schemaPaths[0]); - const snapshot = readSchema(schemaPaths[1]).replace( - /^-- Compatibility snapshot for existing Supabase deployments\.\r?\n-- New PostgreSQL\/RDS deployments must use `npm run db:migrate`; do not use this\r?\n-- file as an unversioned migration source\.\r?\n\r?\n/, - "" - ); +it("keeps the Supabase compatibility snapshot at the latest migrated schema", () => { + const baseline = readSchema(schemaPaths[0]); + const lifecycle = readSchema(lifecycleMigrationPath); + const snapshot = readSchema(schemaPaths[1]); - expect(snapshot).toBe(migration); + for (const fragment of [ + "provider_dispatch_started_at timestamptz", + "dispatch_ready_at timestamptz", + "finalized_at timestamptz", + "generation_jobs_finalize_claim_idx", + "where dispatch_ready_at is not null", + "and finalized_at is null" + ]) { + expect(lifecycle.toLowerCase()).toContain(fragment); + expect(snapshot.toLowerCase()).toContain(fragment); + } + for (const fragment of [ + "create table if not exists assets", + "create table if not exists billing_ledger", + "billing_post_wallet_entry" + ]) { + expect(baseline.toLowerCase()).toContain(fragment); + expect(snapshot.toLowerCase()).toContain(fragment); + } + expect(snapshot).toContain("Compatibility snapshot for existing Supabase deployments"); });