405 lines
16 KiB
Go
405 lines
16 KiB
Go
// Package application composes the Go backend foundation Modules.
|
|
package application
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/httpapi"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/identity"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/localstore"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/orchestration"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/postgres"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/prompt"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/publicapi"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage"
|
|
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/webhook"
|
|
)
|
|
|
|
type Options struct {
|
|
Context context.Context
|
|
Getenv postgres.Getenv
|
|
ReadFile postgres.ReadFile
|
|
// AuthorizationLoader is an optional Identity persistence Adapter used by
|
|
// composition tests and alternate runtime backends. Production defaults to
|
|
// the PostgreSQL Store opened below.
|
|
AuthorizationLoader identity.AuthorizationSnapshotLoader
|
|
// CredentialAuthenticator is the narrow Password Login persistence seam.
|
|
// Production defaults to the same PostgreSQL Store used for authorization.
|
|
CredentialAuthenticator identity.CredentialAuthenticator
|
|
// BlobStore and RemoteFetcher support storage-specific integration tests and
|
|
// alternate deployments. Production defaults to the hardened local store;
|
|
// a fully configured OSS environment is composed below.
|
|
BlobStore assets.BlobStore
|
|
RemoteFetcher assets.RemoteFetcher
|
|
// ProviderRegistry can replace all external adapters in deterministic tests.
|
|
ProviderRegistry jobs.ProviderRegistry
|
|
// Log adapters remain injectable for deterministic composition tests.
|
|
// Runtime settings intentionally have one concrete source so /api/settings
|
|
// and billing account endpoints cannot observe different stores.
|
|
Logs httpapi.LogService
|
|
}
|
|
|
|
type App struct {
|
|
handler http.Handler
|
|
db *postgres.Module
|
|
worker *jobs.WorkerLoop
|
|
}
|
|
|
|
func New(options Options) (*App, error) {
|
|
ctx := options.Context
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
getenv := options.Getenv
|
|
if getenv == nil {
|
|
getenv = os.Getenv
|
|
}
|
|
readFile := options.ReadFile
|
|
if readFile == nil {
|
|
readFile = os.ReadFile
|
|
}
|
|
|
|
authConfig, err := ParseAuthConfig(getenv)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
config, err := postgres.ParseConfig(getenv, readFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if config.Backend == postgres.BackendPostgres && options.ProviderRegistry == nil {
|
|
if err := validateProductionProviderConfiguration(getenv); err != nil {
|
|
return nil, err
|
|
}
|
|
if allowUnconfiguredProviders(getenv) {
|
|
log.Printf("WARNING: %s is enabled; the Go API will start with unconfigured providers, and generation/quote requests will remain unavailable until credentials are added and the process is restarted", "ZHINIAN_ALLOW_UNCONFIGURED_PROVIDERS")
|
|
}
|
|
}
|
|
database, err := postgres.Open(ctx, config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
closeOnError := true
|
|
defer func() {
|
|
if closeOnError {
|
|
database.Close()
|
|
}
|
|
}()
|
|
readiness := databaseReadiness{config: config, store: database.Store}
|
|
|
|
// PostgreSQL remains the production source of truth. Local mode swaps every
|
|
// business persistence port as one coherent process-local unit so modules do
|
|
// not accidentally call the nil-backed PostgreSQL shell used for readiness.
|
|
var authorizationStore identity.AuthorizationSnapshotLoader = database.Store
|
|
var credentialStore identity.CredentialAuthenticator = database.Store
|
|
var passwordChangeStore identity.PasswordChanger = database.Store
|
|
var administrationStore administration.Store = database.Store
|
|
var assetCatalog assets.Catalog = database.Store
|
|
var billingStore billing.Store = database.Store
|
|
var walletPoster billing.WalletPoster = postgres.NewBillingWalletPoster(database.Store)
|
|
var usageRepository usage.Repository = postgres.NewUsageRepository(database.Store)
|
|
var templateCatalog templates.Catalog = database.Store
|
|
var jobStore jobs.Store = database.Store
|
|
var creationState orchestration.CreationStateWriter = database.Store
|
|
var jobState orchestration.JobStateWriter = database.Store
|
|
var settlementState orchestration.SettlementStateWriter = database.Store
|
|
if config.Backend == postgres.BackendLocal {
|
|
store := localstore.New()
|
|
authorizationStore = store
|
|
credentialStore = store
|
|
passwordChangeStore = store
|
|
administrationStore = store
|
|
assetCatalog = store
|
|
billingStore = store
|
|
walletPoster = store
|
|
usageRepository = store
|
|
templateCatalog = store
|
|
jobStore = store
|
|
creationState = store
|
|
jobState = store
|
|
settlementState = store
|
|
}
|
|
var resolver httpapi.SessionResolver
|
|
if authConfig.Configured {
|
|
loader := options.AuthorizationLoader
|
|
if loader == nil {
|
|
loader = authorizationStore
|
|
}
|
|
resolver = identity.NewResolver(loader, authConfig.SessionSecret, "platform", nil)
|
|
}
|
|
authState := httpapi.AuthState{Required: authConfig.Required, Configured: authConfig.Configured}
|
|
platformAuthorizer, err := httpapi.NewPlatformAuthorizer(
|
|
authState,
|
|
resolver,
|
|
httpapi.WithLocalDevelopmentFallback(config.Backend == postgres.BackendLocal && !strings.EqualFold(strings.TrimSpace(getenv("NODE_ENV")), "production")),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
authMe, err := httpapi.NewAuthMeHandler(httpapi.AuthState{
|
|
Required: authConfig.Required, Configured: authConfig.Configured,
|
|
}, resolver)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var passwordIssuer httpapi.PasswordSessionIssuer
|
|
if authConfig.Configured {
|
|
authenticator := options.CredentialAuthenticator
|
|
if authenticator == nil {
|
|
authenticator = credentialStore
|
|
}
|
|
passwordIssuer = identity.NewPasswordLogin(authenticator, nil)
|
|
}
|
|
cookieSecure := getenv("ZHINIAN_AUTH_COOKIE_SECURE")
|
|
publicBaseURL := firstAuthEnv(getenv, "NEXT_PUBLIC_APP_URL", "ZHINIAN_PUBLIC_BASE_URL")
|
|
authPassword, err := httpapi.NewAuthPasswordHandler(httpapi.PasswordAuthConfig{
|
|
Configured: authConfig.Configured,
|
|
SessionSecret: authConfig.SessionSecret,
|
|
CookieSecure: cookieSecure,
|
|
PublicBaseURL: publicBaseURL,
|
|
}, passwordIssuer)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
authLogout := httpapi.NewAuthLogoutHandler(httpapi.LogoutConfig{
|
|
CookieSecure: cookieSecure,
|
|
PublicBaseURL: publicBaseURL,
|
|
})
|
|
authCompatibility := httpapi.NewAuthCompatibilityHandler()
|
|
|
|
publicAuthenticator := publicapi.NewAuthenticator(publicapi.Config{
|
|
APIKeys: getenv("ZHINIAN_API_KEYS"),
|
|
InternalWorkerToken: getenv("ZHINIAN_INTERNAL_WORKER_TOKEN"),
|
|
Production: strings.EqualFold(strings.TrimSpace(getenv("NODE_ENV")), "production"),
|
|
})
|
|
|
|
administrationService := administration.NewService(administrationStore)
|
|
if created, err := BootstrapSuperAdmin(ctx, config.Backend, administrationService, ParseBootstrapAdminConfig(getenv)); err != nil {
|
|
return nil, fmt.Errorf("bootstrap super administrator: %w", err)
|
|
} else if created {
|
|
log.Printf("zhinian-api bootstrapped the first super administrator from ZHINIAN_BOOTSTRAP_ADMIN_* configuration")
|
|
}
|
|
adminHandler, err := httpapi.NewAdminHandler(platformAuthorizer, administrationService)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var passwordChangeHandler http.Handler = unavailableHandler(http.StatusServiceUnavailable)
|
|
if authConfig.Configured {
|
|
passwordChanger := identity.NewPasswordChange(passwordChangeStore, nil)
|
|
passwordChangeHandler, err = httpapi.NewAuthPasswordChangeHandler(httpapi.PasswordChangeConfig{
|
|
SessionSecret: authConfig.SessionSecret, CookieSecure: cookieSecure, PublicBaseURL: publicBaseURL,
|
|
}, platformAuthorizer, passwordChanger)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
blobStore := options.BlobStore
|
|
if blobStore == nil {
|
|
var configured bool
|
|
blobStore, configured, err = configuredOSSBlobStore(getenv)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !configured {
|
|
runtimeDirectory := strings.TrimSpace(getenv("ZHINIAN_RUNTIME_DIR"))
|
|
if runtimeDirectory == "" {
|
|
runtimeDirectory = filepath.Join(".runtime")
|
|
}
|
|
blobStore, err = assets.NewLocalFS(runtimeDirectory, firstNonEmpty(publicBaseURL, "http://127.0.0.1:3000"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
remoteFetcher := options.RemoteFetcher
|
|
if remoteFetcher == nil {
|
|
remoteFetcher, err = assets.NewPublicHTTPRemoteFetcher(
|
|
30*time.Second,
|
|
remoteAssetMaxBytes(getenv),
|
|
assets.NewPublicDestinationPolicy(nil, nil),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
assetService := assets.NewService(assetCatalog, blobStore, remoteFetcher, nil, nil)
|
|
assetsHandler, err := httpapi.NewAssetsHandler(assetService, platformAuthorizer, publicAuthenticator, httpapi.AssetsConfig{
|
|
MaxJSONBytes: positiveInt64Env(getenv, "ZHINIAN_MAX_JSON_BYTES", 1<<20), MaxUploadBytes: positiveInt64Env(getenv, "ZHINIAN_MAX_UPLOAD_BYTES", 20<<20),
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
billingService := billing.NewService(billingStore, nil).SetEnabled(strings.TrimSpace(getenv("ZHINIAN_BILLING_REQUIRED")) != "0")
|
|
runtimeSettings := defaultSettingsService(getenv)
|
|
billingAccounts := settingsBillingAccountStore{service: runtimeSettings}
|
|
templateService := templates.NewService(templateCatalog, nil, nil)
|
|
logService := options.Logs
|
|
if logService == nil {
|
|
logService = defaultLogService(getenv)
|
|
}
|
|
var eventLogger EventLogger
|
|
if candidate, ok := logService.(EventLogger); ok {
|
|
eventLogger = candidate
|
|
}
|
|
miscHandler, err := httpapi.NewMiscHandler(httpapi.MiscDependencies{
|
|
Platform: platformAuthorizer, Templates: templateService, PromptAssembler: prompt.Assemble,
|
|
Settings: runtimeSettings, Logs: logService, Public: publicAuthenticator,
|
|
Capabilities: capabilitySummary(getenv), PublicOrigin: publicBaseURL,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
providerRegistry := options.ProviderRegistry
|
|
if providerRegistry == nil {
|
|
providerRegistry = buildProviderRegistry(getenv)
|
|
}
|
|
var unavailableProviders map[string]string
|
|
if config.Backend == postgres.BackendPostgres {
|
|
unavailableProviders = providerUnavailableMessages(getenv)
|
|
}
|
|
jobService := jobs.NewService(jobStore, nil)
|
|
jobBuilder := jobs.ProviderJobBuilder{
|
|
ImageProvider: imageProvider(getenv), VideoProvider: videoProvider(getenv),
|
|
ImageModel: imageModel(getenv), VideoModel: videoModel(getenv), ImageEngine: imageEngine(getenv), VideoEngine: videoEngine(getenv),
|
|
ImageEngines: providerImageTargets(getenv), VideoEngines: providerVideoTargets(getenv), UnavailableProviders: unavailableProviders, NewID: applicationJobID,
|
|
}
|
|
usageService := usage.Service{
|
|
Repository: usageRepository,
|
|
OrganizationOptions: usage.OrganizationOptionSourceFunc(func(ctx context.Context, requester usage.Requester) ([]usage.Option, error) {
|
|
organizations, listErr := administrationService.ListOrganizations(ctx, administration.Actor{
|
|
ID: requester.AccountID, Role: administration.Role(requester.Role), OrganizationID: requester.OrganizationID,
|
|
})
|
|
if listErr != nil {
|
|
return nil, listErr
|
|
}
|
|
options := make([]usage.Option, len(organizations))
|
|
for index, organization := range organizations {
|
|
options[index] = usage.Option{Value: organization.ID, Label: organization.Name}
|
|
}
|
|
return options, nil
|
|
}),
|
|
}
|
|
usageHandler := httpapi.NewUsageHandler(platformAuthorizer, usageService, nil)
|
|
billingHandler := httpapi.NewBillingHandlerWithBuilder(platformAuthorizer, billingService, billingAccounts, jobBuilder)
|
|
ledger := billing.Ledger{Poster: walletPoster, NewID: func() string { return applicationID("ledger") }}
|
|
creation := orchestration.NewCreationCoordinator(jobBuilder, jobService, billingService, ledger, creationState)
|
|
refunds := orchestration.NewTerminalRefund(ledger, jobState)
|
|
usageRecorder := orchestration.NewUsageRecorder(usageService, func() string { return applicationID("usage") }, nil)
|
|
webhookSender, err := defaultWebhookSender(getenv)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
webhookBridge := orchestration.NewWebhookBridge(webhook.NewDeliverer(webhookSender, getenv("ZHINIAN_WEBHOOK_SECRET"), nil))
|
|
outputs := orchestration.NewAssetOutputRegistrar(assetService, orchestration.ResolveProviderOutputURLs)
|
|
providerProcessor := jobs.ProviderProcessor{Providers: providerRegistry, Store: jobStore}
|
|
settlementProcessor := orchestration.NewSettlementProcessor(providerProcessor, ledger, settlementState, nil)
|
|
processor := orchestration.NewOutputRegisteringProcessor(settlementProcessor, outputs, jobState)
|
|
artifacts := orchestration.NewAssetArtifacts(assetService)
|
|
worker := jobs.NewWorker(jobStore, processor, refunds, usageRecorder, webhookBridge, jobs.WorkerConfig{
|
|
BatchSize: int(positiveInt64Env(getenv, "ZHINIAN_WORKER_BATCH_SIZE", 3)),
|
|
LockTimeoutSeconds: int(positiveInt64Env(getenv, "ZHINIAN_WORKER_LOCK_TIMEOUT_SECONDS", 300)),
|
|
PollInterval: durationEnv(getenv, "ZHINIAN_WORKER_POLL_INTERVAL_MS", 5*time.Second),
|
|
}, nil)
|
|
jobsHandler, err := httpapi.NewJobsHandler(httpapi.JobsDependencies{
|
|
Service: jobService, Platform: platformAuthorizer, Public: publicAuthenticator,
|
|
Builder: httpapi.ProviderBuilderAdapter{Builder: jobBuilder}, Creation: creation, Refunds: refunds, Artifacts: artifacts, Worker: WithTickEventLogging(worker, eventLogger),
|
|
}, httpapi.JobsConfig{MaxJSONBytes: positiveInt64Env(getenv, "ZHINIAN_MAX_JSON_BYTES", 1<<20), NewID: applicationJobID})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var workerLoop *jobs.WorkerLoop
|
|
if parseBool(getenv("ZHINIAN_GO_EMBEDDED_WORKER")) {
|
|
workerLoop = jobs.NewWorkerLoop(WithTickEventLogging(worker, eventLogger), jobs.LoopConfig{
|
|
Interval: durationEnv(getenv, "ZHINIAN_WORKER_POLL_INTERVAL_MS", 5*time.Second), WorkerID: firstNonEmpty(getenv("ZHINIAN_WORKER_ID"), "embedded-worker"),
|
|
})
|
|
workerLoop.Start(ctx)
|
|
}
|
|
|
|
foundation := httpapi.NewHandler(readiness, httpapi.WithHealthDetails(runtimeHealthDetails(getenv)))
|
|
mux := http.NewServeMux()
|
|
mux.Handle("/api/auth/me", authMe)
|
|
mux.Handle("/api/auth/password", authPassword)
|
|
mux.Handle("/api/auth/logout", authLogout)
|
|
mux.Handle("/api/auth/password/change", passwordChangeHandler)
|
|
mux.Handle("/api/auth/login", authCompatibility)
|
|
mux.Handle("/api/auth/callback", authCompatibility)
|
|
mux.Handle("/api/auth/captcha", authCompatibility)
|
|
mux.Handle("/api/admin/accounts", adminHandler)
|
|
mux.Handle("/api/admin/accounts/", adminHandler)
|
|
mux.Handle("/api/admin/organizations", adminHandler)
|
|
mux.Handle("/api/assets", assetsHandler)
|
|
mux.Handle("/api/assets/", assetsHandler)
|
|
mux.Handle("/api/v1/assets", assetsHandler)
|
|
mux.Handle("/api/v1/assets/", assetsHandler)
|
|
mux.Handle("/uploads/", assetsHandler)
|
|
mux.Handle("/generated-results/", assetsHandler)
|
|
mux.Handle("/api/billing", billingHandler)
|
|
mux.Handle("/api/billing/", billingHandler)
|
|
mux.Handle("/api/admin/billing", billingHandler)
|
|
mux.Handle("/api/admin/billing/", billingHandler)
|
|
mux.Handle("/api/usage", usageHandler)
|
|
mux.Handle("/api/admin/usage", usageHandler)
|
|
mux.Handle("/api/generations/", jobsHandler)
|
|
mux.Handle("/api/v1/jobs", jobsHandler)
|
|
mux.Handle("/api/v1/jobs/", jobsHandler)
|
|
mux.Handle("/api/internal/worker/tick", jobsHandler)
|
|
mux.Handle("/api/image-templates", miscHandler)
|
|
mux.Handle("/api/image-templates/", miscHandler)
|
|
mux.Handle("/api/prompt/assemble", miscHandler)
|
|
mux.Handle("/api/settings", miscHandler)
|
|
mux.Handle("/api/logs", miscHandler)
|
|
mux.Handle("/api/v1/capabilities", miscHandler)
|
|
mux.Handle("/api/v1/openapi.json", miscHandler)
|
|
mux.Handle("/", foundation)
|
|
closeOnError = false
|
|
return &App{
|
|
db: database, handler: WithHTTPEventLogging(httpapi.WithRouteMethodCompatibility(mux), eventLogger), worker: workerLoop,
|
|
}, nil
|
|
}
|
|
|
|
func (app *App) Handler() http.Handler {
|
|
return app.handler
|
|
}
|
|
|
|
func (app *App) Close() {
|
|
if app.worker != nil {
|
|
app.worker.Stop()
|
|
}
|
|
if app.db != nil {
|
|
app.db.Close()
|
|
}
|
|
}
|
|
|
|
type databaseReadiness struct {
|
|
config postgres.Config
|
|
store *postgres.Store
|
|
}
|
|
|
|
func (adapter databaseReadiness) Status() httpapi.DatabaseStatus {
|
|
configured := adapter.config.Backend == postgres.BackendLocal || adapter.config.DatabaseURL != ""
|
|
return httpapi.DatabaseStatus{
|
|
Backend: string(adapter.config.Backend),
|
|
Configured: configured,
|
|
}
|
|
}
|
|
|
|
func (adapter databaseReadiness) Ready(ctx context.Context) (httpapi.DatabaseStatus, error) {
|
|
status := adapter.Status()
|
|
return status, adapter.store.Readiness(ctx)
|
|
}
|