Files
NianAIGC/backend/internal/localstore/store.go

551 lines
16 KiB
Go

// Package localstore provides process-local development persistence.
// It is intentionally non-durable and must not be used as a production store.
package localstore
import (
"context"
"encoding/json"
"fmt"
"reflect"
"sort"
"sync"
"time"
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/administration"
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/assets"
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/billing"
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/jobs"
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/templates"
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/usage"
)
const (
DemoAccountID = "demo-merchant"
DemoOrganizationID = "org-demo"
DemoPhone = "13800000000"
DemoPassword = "demo-password"
)
type Option func(*Store)
func WithClock(now func() time.Time) Option {
return func(s *Store) {
if now != nil {
s.now = now
}
}
}
type Store struct {
mu sync.RWMutex
now func() time.Time
accounts map[string]administration.Account
organizations map[string]administration.Organization
assets map[string]assets.Asset
jobs map[string]jobs.Job
templates map[string]templates.Template
usageEvents map[string]usage.Event
usageJobIDs map[string]string
priceRules map[string]billing.PriceRule
wallets map[string]billing.Wallet
ledger []billing.LedgerEntry
postings map[string]walletRecord
}
type walletRecord struct {
params billing.WalletPostParams
posting billing.WalletPosting
}
func New(options ...Option) *Store {
s := &Store{now: time.Now, accounts: map[string]administration.Account{}, organizations: map[string]administration.Organization{}, assets: map[string]assets.Asset{}, jobs: map[string]jobs.Job{}, templates: map[string]templates.Template{}, usageEvents: map[string]usage.Event{}, usageJobIDs: map[string]string{}, priceRules: map[string]billing.PriceRule{}, wallets: map[string]billing.Wallet{}, postings: map[string]walletRecord{}}
for _, option := range options {
option(s)
}
s.seedDemo()
return s
}
func NewStore(options ...Option) *Store { return New(options...) }
func (s *Store) seedDemo() {
now := s.now().UTC()
password, err := administration.HashPassword(DemoPassword)
if err != nil {
panic(fmt.Sprintf("localstore: hash demo password: %v", err))
}
s.organizations[DemoOrganizationID] = administration.Organization{ID: DemoOrganizationID, Name: "演示组织", Status: administration.StatusActive, ArchiveOwnerID: DemoAccountID, CreatedAt: now, UpdatedAt: now}
s.accounts[DemoAccountID] = administration.Account{ID: DemoAccountID, Phone: DemoPhone, DisplayName: "智念用户", Role: administration.RoleSuperAdmin, OrganizationID: DemoOrganizationID, Status: administration.StatusActive, PasswordHash: password.Hash, PasswordSalt: password.Salt, SessionVersion: 1, CreatedAt: now, UpdatedAt: now}
}
func cloneAsset(a assets.Asset) assets.Asset {
a.Tags = append([]string(nil), a.Tags...)
a.Metadata = cloneMap(a.Metadata)
return a
}
func cloneJob(j jobs.Job) jobs.Job {
j.InputAssetIDs = append([]string(nil), j.InputAssetIDs...)
j.InputURLs = append([]string(nil), j.InputURLs...)
j.OutputAssetIDs = append([]string(nil), j.OutputAssetIDs...)
j.RequestPayload = append(json.RawMessage(nil), j.RequestPayload...)
j.ResponsePayload = append(json.RawMessage(nil), j.ResponsePayload...)
j.WebhookLastStatus = append(json.RawMessage(nil), j.WebhookLastStatus...)
j.UsageContext = append(json.RawMessage(nil), j.UsageContext...)
j.Billing = append(json.RawMessage(nil), j.Billing...)
if j.Error != nil {
e := *j.Error
j.Error = &e
}
if j.ProviderDispatchStartedAt != nil {
value := *j.ProviderDispatchStartedAt
j.ProviderDispatchStartedAt = &value
}
if j.DispatchReadyAt != nil {
value := *j.DispatchReadyAt
j.DispatchReadyAt = &value
}
if j.FinalizedAt != nil {
value := *j.FinalizedAt
j.FinalizedAt = &value
}
return j
}
func cloneMap(m map[string]any) map[string]any {
if m == nil {
return nil
}
raw, err := json.Marshal(m)
if err != nil {
out := make(map[string]any, len(m))
for k, v := range m {
out[k] = v
}
return out
}
var out map[string]any
_ = json.Unmarshal(raw, &out)
return out
}
func cloneRule(r billing.PriceRule) billing.PriceRule {
r.Conditions = billing.Conditions(cloneMap(map[string]any(r.Conditions)))
r.Source = cloneMap(r.Source)
raw, _ := json.Marshal(r.Dimensions)
_ = json.Unmarshal(raw, &r.Dimensions)
return r
}
// Assets.
func (s *Store) ListOwner(_ context.Context, owner string) ([]assets.Asset, error) {
s.mu.RLock()
defer s.mu.RUnlock()
out := []assets.Asset{}
for _, a := range s.assets {
if a.OwnerID == owner {
out = append(out, cloneAsset(a))
}
}
sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
return out, nil
}
func (s *Store) GetOwner(_ context.Context, owner, id string) (assets.Asset, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
a, ok := s.assets[id]
if !ok || a.OwnerID != owner {
return assets.Asset{}, false, nil
}
return cloneAsset(a), true, nil
}
func (s *Store) GetOwnerByStoragePath(_ context.Context, owner, path string) (assets.Asset, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
for _, a := range s.assets {
if a.OwnerID == owner && a.StoragePath == path {
return cloneAsset(a), true, nil
}
}
return assets.Asset{}, false, nil
}
func (s *Store) ListPublic(_ context.Context, owner, client string, limit int) ([]assets.Asset, error) {
s.mu.RLock()
defer s.mu.RUnlock()
allowed := s.publicAssetIDs(owner, client, limit)
tag := assets.ClientTag(client)
out := []assets.Asset{}
for _, a := range s.assets {
if a.OwnerID == owner && (contains(a.Tags, tag) || allowed[a.ID]) {
out = append(out, cloneAsset(a))
}
}
sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
return out, nil
}
func (s *Store) GetPublic(_ context.Context, owner, client, id string, limit int) (assets.Asset, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
a, ok := s.assets[id]
if !ok || a.OwnerID != owner {
return assets.Asset{}, false, nil
}
if !contains(a.Tags, assets.ClientTag(client)) && !s.publicAssetIDs(owner, client, limit)[id] {
return assets.Asset{}, false, nil
}
return cloneAsset(a), true, nil
}
func (s *Store) publicAssetIDs(owner, client string, limit int) map[string]bool {
candidates := []jobs.Job{}
for _, j := range s.jobs {
if j.OwnerID == owner && j.ExternalClientID == client {
candidates = append(candidates, j)
}
}
sort.Slice(candidates, func(i, j int) bool { return candidates[i].CreatedAt.After(candidates[j].CreatedAt) })
if limit <= 0 {
limit = 200
}
if len(candidates) > limit {
candidates = candidates[:limit]
}
out := map[string]bool{}
for _, j := range candidates {
for _, id := range append(append([]string{}, j.InputAssetIDs...), j.OutputAssetIDs...) {
out[id] = true
}
}
return out
}
func (s *Store) Create(_ context.Context, a assets.Asset) (assets.Asset, error) {
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.assets[a.ID]; ok {
return assets.Asset{}, fmt.Errorf("asset already exists: %s", a.ID)
}
if a.Tags == nil {
a.Tags = []string{}
}
if a.Metadata == nil {
a.Metadata = map[string]any{}
}
s.assets[a.ID] = cloneAsset(a)
return cloneAsset(a), nil
}
func (s *Store) DeleteOwner(_ context.Context, owner, id string) (assets.Asset, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
a, ok := s.assets[id]
if !ok || a.OwnerID != owner {
return assets.Asset{}, false, nil
}
delete(s.assets, id)
return cloneAsset(a), true, nil
}
// Jobs.
func (s *Store) ListJobs(_ context.Context, f jobs.ListFilter) ([]jobs.Job, error) {
s.mu.RLock()
defer s.mu.RUnlock()
out := []jobs.Job{}
for _, j := range s.jobs {
if !f.Scope.Owns(j) || f.Status != "" && j.Status != f.Status || f.Capability != "" && j.Capability != f.Capability || f.Before != nil && !j.CreatedAt.Before(*f.Before) {
continue
}
out = append(out, cloneJob(j))
}
sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
if f.Limit > 0 && len(out) > f.Limit {
out = out[:f.Limit]
}
return out, nil
}
func (s *Store) FindJob(_ context.Context, scope jobs.Scope, id string) (jobs.Job, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
j, ok := s.jobs[id]
if !ok || !scope.Owns(j) {
return jobs.Job{}, false, nil
}
return cloneJob(j), true, nil
}
func (s *Store) FindIdempotentJob(_ context.Context, scope jobs.Scope, key string) (jobs.Job, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
for _, j := range s.jobs {
if scope.Owns(j) && j.IdempotencyKey == key {
return cloneJob(j), true, nil
}
}
return jobs.Job{}, false, nil
}
func (s *Store) CreateJob(_ context.Context, j jobs.Job) (jobs.Job, error) {
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.jobs[j.ID]; ok {
return jobs.Job{}, fmt.Errorf("generation job already exists: %s", j.ID)
}
if j.ExternalClientID != "" && j.IdempotencyKey != "" {
for _, v := range s.jobs {
if v.OwnerID == j.OwnerID && v.ExternalClientID == j.ExternalClientID && v.IdempotencyKey == j.IdempotencyKey {
return jobs.Job{}, jobs.ErrUniqueIdempotency
}
}
}
s.jobs[j.ID] = cloneJob(j)
return cloneJob(j), nil
}
func (s *Store) UpdateJob(_ context.Context, id string, p jobs.Patch) (jobs.Job, error) {
s.mu.Lock()
defer s.mu.Unlock()
j, ok := s.jobs[id]
if !ok {
return jobs.Job{}, fmt.Errorf("generation job not found: %s", id)
}
if len(p.ExpectedStatuses) > 0 && !containsStatus(p.ExpectedStatuses, j.Status) {
return jobs.Job{}, jobs.ErrTransitionConflict
}
if p.ExpectedLockedBy != nil && j.LockedBy != *p.ExpectedLockedBy {
return jobs.Job{}, jobs.ErrTransitionConflict
}
if p.Status != nil {
j.Status = *p.Status
}
if p.Error != nil {
e := *p.Error
j.Error = &e
}
if p.ClearError {
j.Error = nil
}
if p.Attempts != nil {
j.Attempts = *p.Attempts
}
if p.ScheduledAt != nil {
j.ScheduledAt = *p.ScheduledAt
}
if p.CompletedAt != nil {
v := *p.CompletedAt
j.CompletedAt = &v
}
if p.ProviderTaskID != nil {
j.ProviderTaskID = *p.ProviderTaskID
}
if p.ProviderDispatchStartedAt != nil {
started := *p.ProviderDispatchStartedAt
j.ProviderDispatchStartedAt = &started
}
if p.ClearProviderDispatch {
j.ProviderDispatchStartedAt = nil
}
if p.SetResponsePayload {
j.ResponsePayload = append([]byte(nil), p.ResponsePayload...)
}
if p.DispatchReadyAt != nil {
ready := *p.DispatchReadyAt
j.DispatchReadyAt = &ready
}
if p.FinalizedAt != nil {
finalized := *p.FinalizedAt
j.FinalizedAt = &finalized
}
if p.ClearProviderTaskID {
j.ProviderTaskID = ""
}
if p.ClearLease {
j.LockedAt = nil
j.LockedBy = ""
}
if p.WebhookAttempts != nil {
j.WebhookAttempts = *p.WebhookAttempts
}
if p.SetWebhookStatus {
j.WebhookLastStatus = append([]byte(nil), p.WebhookLastStatus...)
}
j.UpdatedAt = s.now().UTC()
s.jobs[id] = cloneJob(j)
return cloneJob(j), nil
}
func (s *Store) DeleteJob(_ context.Context, id string) error {
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.jobs[id]; !ok {
return fmt.Errorf("generation job not found: %s", id)
}
delete(s.jobs, id)
return nil
}
func (s *Store) ClaimJobs(_ context.Context, worker string, limit, timeoutSeconds int) ([]jobs.Job, error) {
s.mu.Lock()
defer s.mu.Unlock()
if limit < 1 {
limit = 1
}
if limit > 20 {
limit = 20
}
if timeoutSeconds <= 0 {
timeoutSeconds = 300
}
now := s.now().UTC()
eligible := []jobs.Job{}
for _, j := range s.jobs {
if j.DispatchReadyAt == nil || j.FinalizedAt != nil || j.ScheduledAt.After(now) {
continue
}
leaseAvailable := j.LockedAt == nil || j.LockedAt.Add(time.Duration(timeoutSeconds)*time.Second).Before(now)
if leaseAvailable && (j.Status == jobs.StatusQueued || j.Status == jobs.StatusRunning || j.Status.Terminal()) {
eligible = append(eligible, j)
}
}
sort.Slice(eligible, func(i, j int) bool {
if eligible[i].Priority != eligible[j].Priority {
return eligible[i].Priority > eligible[j].Priority
}
if !eligible[i].ScheduledAt.Equal(eligible[j].ScheduledAt) {
return eligible[i].ScheduledAt.Before(eligible[j].ScheduledAt)
}
return eligible[i].CreatedAt.Before(eligible[j].CreatedAt)
})
if len(eligible) > limit {
eligible = eligible[:limit]
}
out := make([]jobs.Job, 0, len(eligible))
for _, j := range eligible {
j.LockedBy = worker
j.LockedAt = timePtr(now)
if j.StartedAt == nil {
j.StartedAt = timePtr(now)
}
j.UpdatedAt = now
s.jobs[j.ID] = cloneJob(j)
out = append(out, cloneJob(j))
}
return out, nil
}
func (s *Store) WriteBilling(_ context.Context, id string, value json.RawMessage) error {
if len(value) == 0 || !json.Valid(value) {
return fmt.Errorf("write generation billing: invalid JSON")
}
s.mu.Lock()
defer s.mu.Unlock()
j, ok := s.jobs[id]
if !ok {
return fmt.Errorf("write generation billing: generation job not found")
}
j.Billing = append([]byte(nil), value...)
j.UpdatedAt = s.now().UTC()
s.jobs[id] = j
return nil
}
func (s *Store) WriteBillingFenced(_ context.Context, id string, value json.RawMessage, status jobs.Status, lockedBy string) error {
if len(value) == 0 || !json.Valid(value) {
return fmt.Errorf("write generation billing: invalid JSON")
}
s.mu.Lock()
defer s.mu.Unlock()
j, ok := s.jobs[id]
if !ok || j.Status != status || j.LockedBy != lockedBy {
return jobs.ErrTransitionConflict
}
j.Billing = append([]byte(nil), value...)
j.UpdatedAt = s.now().UTC()
s.jobs[id] = cloneJob(j)
return nil
}
func (s *Store) ActivateCreation(_ context.Context, id string, value json.RawMessage) error {
if len(value) == 0 || !json.Valid(value) {
return fmt.Errorf("activate generation creation: invalid JSON")
}
s.mu.Lock()
defer s.mu.Unlock()
j, ok := s.jobs[id]
if !ok {
return fmt.Errorf("activate generation creation: generation job not found")
}
if j.DispatchReadyAt != nil {
if !jsonEqual(j.Billing, value) {
return jobs.ErrTransitionConflict
}
return nil
}
j.Billing = append([]byte(nil), value...)
ready := s.now().UTC()
j.DispatchReadyAt = &ready
j.UpdatedAt = s.now().UTC()
s.jobs[id] = cloneJob(j)
return nil
}
func jsonEqual(a, b json.RawMessage) bool {
var left, right any
return json.Unmarshal(a, &left) == nil && json.Unmarshal(b, &right) == nil && reflect.DeepEqual(left, right)
}
func (s *Store) WriteOutputAssetIDs(_ context.Context, id string, values []string) error {
s.mu.Lock()
defer s.mu.Unlock()
j, ok := s.jobs[id]
if !ok {
return fmt.Errorf("write generation output assets: generation job not found")
}
j.OutputAssetIDs = append([]string{}, values...)
j.UpdatedAt = s.now().UTC()
s.jobs[id] = j
return nil
}
func (s *Store) WriteOutputAssetIDsFenced(_ context.Context, id string, values []string, status jobs.Status, lockedBy string) error {
s.mu.Lock()
defer s.mu.Unlock()
j, ok := s.jobs[id]
if !ok || j.Status != status || j.LockedBy != lockedBy {
return jobs.ErrTransitionConflict
}
j.OutputAssetIDs = append([]string{}, values...)
j.UpdatedAt = s.now().UTC()
s.jobs[id] = cloneJob(j)
return nil
}
func (s *Store) FailCreation(_ context.Context, j jobs.Job) error {
if j.Status != jobs.StatusFailed || j.Error == nil || len(j.Billing) == 0 || !json.Valid(j.Billing) {
return fmt.Errorf("fail generation creation: invalid terminal state")
}
s.mu.Lock()
defer s.mu.Unlock()
current, ok := s.jobs[j.ID]
if !ok {
return fmt.Errorf("fail generation creation: generation job not found")
}
current.Status = j.Status
e := *j.Error
current.Error = &e
current.Billing = append([]byte(nil), j.Billing...)
now := s.now().UTC()
if current.CompletedAt == nil {
current.CompletedAt = &now
}
current.LockedAt = nil
current.LockedBy = ""
current.UpdatedAt = now
s.jobs[j.ID] = current
return nil
}
func timePtr(v time.Time) *time.Time { return &v }
func contains(values []string, want string) bool {
for _, v := range values {
if v == want {
return true
}
}
return false
}
func containsStatus(values []jobs.Status, want jobs.Status) bool {
for _, value := range values {
if value == want {
return true
}
}
return false
}