oss请求处理

This commit is contained in:
andy committed 2026-08-19 10:52:55 +08:00
1 parent 18cb51670c
commit a9a4cdf125
18 files changed
+450 -29

No files matched your search

+55 -2
View File
@@ -20,13 +20,19 @@ type ProviderResolver interface {
Resolve(context.Context, string) (providers.Adapter, error)
}
type ProviderAssetURLResolver interface {
ResolveProviderAssetURL(context.Context, string, string, time.Duration) (sourceURL string, accessURL string, err error)
}
func (registry ProviderRegistry) Resolve(_ context.Context, name string) (providers.Adapter, error) {
return registry[name], nil
}
type ProviderProcessor struct {
Providers ProviderResolver
Store Store
Providers ProviderResolver
Store Store
AssetURLs ProviderAssetURLResolver
AssetURLTTL time.Duration
}
func (p ProviderProcessor) Advance(ctx context.Context, job Job) (Job, error) {
@@ -49,6 +55,12 @@ func (p ProviderProcessor) Advance(ctx context.Context, job Job) (Job, error) {
if err := json.Unmarshal(job.RequestPayload, &request); err != nil {
return Job{}, errors.New("invalid provider request payload")
}
if job.ProviderTaskID == "" && job.ProviderDispatchStartedAt == nil {
request, err = p.refreshAssetURLs(ctx, job, request)
if err != nil {
return Job{}, fmt.Errorf("prepare provider asset URLs: %w", err)
}
}
var result providers.Result
expectedStatus := job.Status
if job.ProviderTaskID == "" {
@@ -128,6 +140,47 @@ func (p ProviderProcessor) Advance(ctx context.Context, job Job) (Job, error) {
return job, nil
}
func (p ProviderProcessor) refreshAssetURLs(ctx context.Context, job Job, request providers.Request) (providers.Request, error) {
if p.AssetURLs == nil || len(job.InputAssetIDs) == 0 {
return request, nil
}
ttl := p.AssetURLTTL
if ttl <= 0 {
ttl = time.Hour
}
replacements := make(map[string]string, len(job.InputAssetIDs))
seen := make(map[string]struct{}, len(job.InputAssetIDs))
for _, assetID := range job.InputAssetIDs {
assetID = strings.TrimSpace(assetID)
if assetID == "" {
continue
}
if _, ok := seen[assetID]; ok {
continue
}
seen[assetID] = struct{}{}
sourceURL, accessURL, err := p.AssetURLs.ResolveProviderAssetURL(ctx, job.OwnerID, assetID, ttl)
if err != nil {
return providers.Request{}, err
}
if strings.TrimSpace(sourceURL) == "" || strings.TrimSpace(accessURL) == "" {
return providers.Request{}, errors.New("provider asset URL is unavailable")
}
replacements[sourceURL] = accessURL
}
for index, inputURL := range request.InputURLs {
if accessURL, ok := replacements[inputURL]; ok {
request.InputURLs[index] = accessURL
}
}
for index := range request.Materials {
if accessURL, ok := replacements[request.Materials[index].URL]; ok {
request.Materials[index].URL = accessURL
}
}
return request, nil
}
type ProviderJobBuilder struct {
ImageProvider, VideoProvider string
ImageModel, VideoModel string
+43 -1
View File
@@ -203,6 +203,35 @@ func TestProviderBuilderAndProcessor(t *testing.T) {
}
}
func TestProviderProcessorRefreshesPrivateAssetURLImmediatelyBeforeSubmit(t *testing.T) {
store := newMemoryJobStore()
job := Job{
ID: "job-private", OwnerID: "owner-a", Provider: "fixture", ReqKey: "model-a", Capability: "image.generate", Status: StatusQueued,
InputAssetIDs: []string{"asset-private"}, InputURLs: []string{"https://private.test/uploads/a.png"},
RequestPayload: json.RawMessage(`{"capability":"image.generate","model":"model-a","prompt":"draw","inputUrls":["https://private.test/uploads/a.png"],"materials":[{"url":"https://private.test/uploads/a.png","type":"image"}]}`),
}
store.jobs[job.ID] = job
adapter := &countingProvider{result: providers.Result{TaskID: "provider-task", Status: providers.StatusQueued}}
resolver := &recordingProviderAssetURLResolver{
sourceURL: "https://private.test/uploads/a.png",
accessURL: "https://private.test/uploads/a.png?OSSAccessKeyId=test&Signature=temporary",
}
processor := ProviderProcessor{Providers: ProviderRegistry{"fixture": adapter}, Store: store, AssetURLs: resolver}
if _, err := processor.Advance(context.Background(), job); err != nil {
t.Fatal(err)
}
if resolver.owner != "owner-a" || resolver.assetID != "asset-private" || resolver.ttl != time.Hour {
t.Fatalf("resolver call = owner %q asset %q ttl %s", resolver.owner, resolver.assetID, resolver.ttl)
}
if !reflect.DeepEqual(adapter.request.InputURLs, []string{resolver.accessURL}) || len(adapter.request.Materials) != 1 || adapter.request.Materials[0].URL != resolver.accessURL {
t.Fatalf("submitted request = %#v", adapter.request)
}
if strings.Contains(string(job.RequestPayload), "Signature=temporary") || strings.Contains(string(store.jobs[job.ID].RequestPayload), "Signature=temporary") {
t.Fatal("temporary signature must not be persisted in the job request")
}
}
func TestProviderProcessorNeverResubmitsAfterPersistedDispatchIntent(t *testing.T) {
store := newMemoryJobStore()
started := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
@@ -290,6 +319,18 @@ func TestProviderJobBuilderClampsPriorityToPublicContract(t *testing.T) {
type countingProvider struct {
submits int
result providers.Result
request providers.Request
}
type recordingProviderAssetURLResolver struct {
sourceURL, accessURL string
owner, assetID string
ttl time.Duration
}
func (resolver *recordingProviderAssetURLResolver) ResolveProviderAssetURL(_ context.Context, owner, assetID string, ttl time.Duration) (string, string, error) {
resolver.owner, resolver.assetID, resolver.ttl = owner, assetID, ttl
return resolver.sourceURL, resolver.accessURL, nil
}
type modelQueryProvider struct {
@@ -308,8 +349,9 @@ func (provider *modelQueryProvider) QueryModel(_ context.Context, _ string, mode
return provider.result, nil
}
func (p *countingProvider) Submit(context.Context, providers.Request) (providers.Result, error) {
func (p *countingProvider) Submit(_ context.Context, request providers.Request) (providers.Result, error) {
p.submits++
p.request = request
return p.result, nil
}
func (p *countingProvider) Query(context.Context, string) (providers.Result, error) {