oss请求处理
This commit is contained in:
1 parent
18cb51670c
commit
a9a4cdf125
18 files changed
+450
-29
No files matched your search
@@ -20,13 +20,19 @@ type ProviderResolver interface {
|
||||
Resolve(context.Context, string) (providers.Adapter, error)
|
||||
}
|
||||
|
||||
type ProviderAssetURLResolver interface {
|
||||
ResolveProviderAssetURL(context.Context, string, string, time.Duration) (sourceURL string, accessURL string, err error)
|
||||
}
|
||||
|
||||
func (registry ProviderRegistry) Resolve(_ context.Context, name string) (providers.Adapter, error) {
|
||||
return registry[name], nil
|
||||
}
|
||||
|
||||
type ProviderProcessor struct {
|
||||
Providers ProviderResolver
|
||||
Store Store
|
||||
Providers ProviderResolver
|
||||
Store Store
|
||||
AssetURLs ProviderAssetURLResolver
|
||||
AssetURLTTL time.Duration
|
||||
}
|
||||
|
||||
func (p ProviderProcessor) Advance(ctx context.Context, job Job) (Job, error) {
|
||||
@@ -49,6 +55,12 @@ func (p ProviderProcessor) Advance(ctx context.Context, job Job) (Job, error) {
|
||||
if err := json.Unmarshal(job.RequestPayload, &request); err != nil {
|
||||
return Job{}, errors.New("invalid provider request payload")
|
||||
}
|
||||
if job.ProviderTaskID == "" && job.ProviderDispatchStartedAt == nil {
|
||||
request, err = p.refreshAssetURLs(ctx, job, request)
|
||||
if err != nil {
|
||||
return Job{}, fmt.Errorf("prepare provider asset URLs: %w", err)
|
||||
}
|
||||
}
|
||||
var result providers.Result
|
||||
expectedStatus := job.Status
|
||||
if job.ProviderTaskID == "" {
|
||||
@@ -128,6 +140,47 @@ func (p ProviderProcessor) Advance(ctx context.Context, job Job) (Job, error) {
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (p ProviderProcessor) refreshAssetURLs(ctx context.Context, job Job, request providers.Request) (providers.Request, error) {
|
||||
if p.AssetURLs == nil || len(job.InputAssetIDs) == 0 {
|
||||
return request, nil
|
||||
}
|
||||
ttl := p.AssetURLTTL
|
||||
if ttl <= 0 {
|
||||
ttl = time.Hour
|
||||
}
|
||||
replacements := make(map[string]string, len(job.InputAssetIDs))
|
||||
seen := make(map[string]struct{}, len(job.InputAssetIDs))
|
||||
for _, assetID := range job.InputAssetIDs {
|
||||
assetID = strings.TrimSpace(assetID)
|
||||
if assetID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[assetID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[assetID] = struct{}{}
|
||||
sourceURL, accessURL, err := p.AssetURLs.ResolveProviderAssetURL(ctx, job.OwnerID, assetID, ttl)
|
||||
if err != nil {
|
||||
return providers.Request{}, err
|
||||
}
|
||||
if strings.TrimSpace(sourceURL) == "" || strings.TrimSpace(accessURL) == "" {
|
||||
return providers.Request{}, errors.New("provider asset URL is unavailable")
|
||||
}
|
||||
replacements[sourceURL] = accessURL
|
||||
}
|
||||
for index, inputURL := range request.InputURLs {
|
||||
if accessURL, ok := replacements[inputURL]; ok {
|
||||
request.InputURLs[index] = accessURL
|
||||
}
|
||||
}
|
||||
for index := range request.Materials {
|
||||
if accessURL, ok := replacements[request.Materials[index].URL]; ok {
|
||||
request.Materials[index].URL = accessURL
|
||||
}
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
type ProviderJobBuilder struct {
|
||||
ImageProvider, VideoProvider string
|
||||
ImageModel, VideoModel string
|
||||
|
||||
@@ -203,6 +203,35 @@ func TestProviderBuilderAndProcessor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderProcessorRefreshesPrivateAssetURLImmediatelyBeforeSubmit(t *testing.T) {
|
||||
store := newMemoryJobStore()
|
||||
job := Job{
|
||||
ID: "job-private", OwnerID: "owner-a", Provider: "fixture", ReqKey: "model-a", Capability: "image.generate", Status: StatusQueued,
|
||||
InputAssetIDs: []string{"asset-private"}, InputURLs: []string{"https://private.test/uploads/a.png"},
|
||||
RequestPayload: json.RawMessage(`{"capability":"image.generate","model":"model-a","prompt":"draw","inputUrls":["https://private.test/uploads/a.png"],"materials":[{"url":"https://private.test/uploads/a.png","type":"image"}]}`),
|
||||
}
|
||||
store.jobs[job.ID] = job
|
||||
adapter := &countingProvider{result: providers.Result{TaskID: "provider-task", Status: providers.StatusQueued}}
|
||||
resolver := &recordingProviderAssetURLResolver{
|
||||
sourceURL: "https://private.test/uploads/a.png",
|
||||
accessURL: "https://private.test/uploads/a.png?OSSAccessKeyId=test&Signature=temporary",
|
||||
}
|
||||
processor := ProviderProcessor{Providers: ProviderRegistry{"fixture": adapter}, Store: store, AssetURLs: resolver}
|
||||
|
||||
if _, err := processor.Advance(context.Background(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resolver.owner != "owner-a" || resolver.assetID != "asset-private" || resolver.ttl != time.Hour {
|
||||
t.Fatalf("resolver call = owner %q asset %q ttl %s", resolver.owner, resolver.assetID, resolver.ttl)
|
||||
}
|
||||
if !reflect.DeepEqual(adapter.request.InputURLs, []string{resolver.accessURL}) || len(adapter.request.Materials) != 1 || adapter.request.Materials[0].URL != resolver.accessURL {
|
||||
t.Fatalf("submitted request = %#v", adapter.request)
|
||||
}
|
||||
if strings.Contains(string(job.RequestPayload), "Signature=temporary") || strings.Contains(string(store.jobs[job.ID].RequestPayload), "Signature=temporary") {
|
||||
t.Fatal("temporary signature must not be persisted in the job request")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderProcessorNeverResubmitsAfterPersistedDispatchIntent(t *testing.T) {
|
||||
store := newMemoryJobStore()
|
||||
started := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
@@ -290,6 +319,18 @@ func TestProviderJobBuilderClampsPriorityToPublicContract(t *testing.T) {
|
||||
type countingProvider struct {
|
||||
submits int
|
||||
result providers.Result
|
||||
request providers.Request
|
||||
}
|
||||
|
||||
type recordingProviderAssetURLResolver struct {
|
||||
sourceURL, accessURL string
|
||||
owner, assetID string
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
func (resolver *recordingProviderAssetURLResolver) ResolveProviderAssetURL(_ context.Context, owner, assetID string, ttl time.Duration) (string, string, error) {
|
||||
resolver.owner, resolver.assetID, resolver.ttl = owner, assetID, ttl
|
||||
return resolver.sourceURL, resolver.accessURL, nil
|
||||
}
|
||||
|
||||
type modelQueryProvider struct {
|
||||
@@ -308,8 +349,9 @@ func (provider *modelQueryProvider) QueryModel(_ context.Context, _ string, mode
|
||||
return provider.result, nil
|
||||
}
|
||||
|
||||
func (p *countingProvider) Submit(context.Context, providers.Request) (providers.Result, error) {
|
||||
func (p *countingProvider) Submit(_ context.Context, request providers.Request) (providers.Result, error) {
|
||||
p.submits++
|
||||
p.request = request
|
||||
return p.result, nil
|
||||
}
|
||||
func (p *countingProvider) Query(context.Context, string) (providers.Result, error) {
|
||||
|
||||
Reference in new issue
Block a user