From 809364ecbed9119003ea8e4dbb9325850d704ccf Mon Sep 17 00:00:00 2001 From: brother7 <7brother7@gmail.com> Date: Wed, 12 Aug 2026 20:45:24 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20AI=20=E8=AE=BE=E8=AE=A1?= =?UTF-8?q?=E5=9B=BE=E7=94=9F=E5=9B=BE=E9=80=89=E5=9B=BE=E4=BA=A4=E4=BA=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...60812-design-image-to-image-client-c91e.md | 92 +++++++++++ README.md | 4 +- src/pages/ImageCanvas/index.tsx | 125 ++++++++------- tests/unit/image-canvas-page.test.tsx | 148 ++++++++++++++++++ 4 files changed, 312 insertions(+), 57 deletions(-) create mode 100644 .project-docs/30-worklog/tasks/20260812-design-image-to-image-client-c91e.md diff --git a/.project-docs/30-worklog/tasks/20260812-design-image-to-image-client-c91e.md b/.project-docs/30-worklog/tasks/20260812-design-image-to-image-client-c91e.md new file mode 100644 index 0000000..2e7465c --- /dev/null +++ b/.project-docs/30-worklog/tasks/20260812-design-image-to-image-client-c91e.md @@ -0,0 +1,92 @@ +# Task: 支持 AI Design 图生图选图交互 + +## Identity + +- Task ID: 20260812-design-image-to-image-client-c91e +- Mode: Feature +- Branch: codex/20260812-design-image-to-image-client-c91e-design-image-to-image +- Worktree: D:\Datas\OthersProjects\makelore-design-image-to-image-client-c91e +- Base commit: 3a80625fe2fb74ab4e0a883188a52eade8cd23b9 +- Owner: codex +- Status: Completed + +## Scope + +- Generalize the existing video-first-frame picker into a single-image source + picker that supports both image reference generation and video first frames. +- Reuse existing workspace generated-image selection, local upload and + `attachmentAssetIds` transport without adding a parallel upload path. +- Send medium-correct user intent text to the Design Agent and cover both + interactions with focused page tests. + +## Intent And Constraints + +- Image-to-image accepts exactly one image from the current workspace or one + local upload; task and asset ownership remain Workspace-scoped. +- Do not alter Workspace / Conversation / Task ownership or event routing. +- Preserve the currently deployed string quick-reply trigger for compatibility, + while removing video-only labels and submission semantics from the picker. +- A structured picker action is preferable long-term, but is outside this + server-compatible increment and is recorded as a follow-up. + +## Outcome + +- The former video-only first-frame dialog is now a reusable single-image source + picker. For an image Brief it presents “选择图生图参考图” and sends one selected + Workspace result Asset or one uploaded Asset through the existing + `attachmentAssetIds` Turn transport; for a video Brief it retains the original + first-frame wording and behavior. +- The picker closes as soon as a completed work is selected and after a local + upload succeeds, while request failures continue to surface through the + existing design-conversation error path. +- Only an explicit `brief.medium === 'image'` selects image-reference semantics; + `video`, null and missing medium values default to the historical video + behavior for compatibility with delayed or older server snapshots. +- The deployed exact quick reply “从作品列表选择图片” remains supported. README + now documents text-to-image, single-reference image-to-image, project-work + selection and local upload; no parallel upload API or client state model was + introduced. + +## Verification + +- `pnpm exec vitest run tests/unit/image-canvas-page.test.tsx`: `25 passed`. +- `pnpm run typecheck`: passed. +- `pnpm exec eslint src/pages/ImageCanvas/index.tsx + tests/unit/image-canvas-page.test.tsx`: passed. +- `pnpm run build:vite`: passed; only the repository's existing dynamic-import + and chunk-size warnings were emitted. +- `git diff --check`: passed. +- Independent read-only Sol review: `PASS`, with no P0-P3 findings after + inspecting both the client and matching server diffs; the reviewer reran this + 25-test page suite. +- No packaged desktop smoke or live Works Square provider request was performed. + +## Follow-ups + +- Evolve `DesignMessage.quickReplies` from display-only strings to structured + entries such as `select_image_asset` plus + `image_reference | video_first_frame`. The current release preserves the + deployed exact Chinese string for compatibility, but localization or wording + changes remain coupled to behavior until the protocol evolves. +- Add a first-class “remove reference image” control if the server introduces an + explicit clear-source action; omission of an attachment must not implicitly + clear a Conversation's frozen reference. + +## Promotion Candidates + +- Target canonical documents: `20-architecture/system-overview.md`, + `20-architecture/module-map.md`, `20-architecture/data-flow.md`, and + `40-domain/business-rules.md`. Proposal: record the generalized single-image + source picker, its single-Asset attachment boundary, medium-specific intent, + and null-medium video compatibility fallback. Evidence: the 25 focused page + tests plus typecheck, scoped ESLint and production build listed above. Future + impact: future picker redesigns must preserve Workspace ownership and attach + one real Asset ID rather than relying on a visual description. No semantic + conflict; canonical promotion belongs to a serialized Integration task. +- Target canonical ADR/protocol documentation: the accepted AI Design + Conversation/Gateway contract. Proposal: replace localized quick-reply string + matching with a structured picker action before adding more asset-input + purposes. Evidence: one display string now intentionally opens both image and + video source flows, with purpose inferred from current Brief medium. Future + impact: removes localization and delayed-state coupling. This is a future + protocol change and requires coordinated server/client design confirmation. diff --git a/README.md b/README.md index 96c9009..b80ef1b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Makelore 是一个面向软件与视觉创作的 AI 桌面工作台。当前版本为 `2.0.0`，包含两个已开通产品模块，并保留一个学习模块入口： - `Makelore Code｜AI 编程`：管理本地项目、项目 Agent、会话、文件上下文、代码变更和运行时，并为服务端可运行版本提供真机扫码预览。 -- `Makelore Canvas｜AI 绘画`：以设计项目（Workspace）组织 Agent 对话、方向确认、图片/视频生成任务和私有结果。 +- `Makelore Canvas｜AI 绘画`：以设计项目（Workspace）组织 Agent 对话、方向确认、文生图、单参考图生图、视频生成任务和私有结果；参考图可从当前项目作品选择或从本地上传。 - `Makelore Learning｜AI 学习`：数学与知识宇宙主题的学习入口，当前暂未开通；开屏页和模块切换菜单保留置灰入口，不能进入。 作品广场、素材广场、独立发布上传和云部署页面不属于 Makelore 2.0 工作台。新建项目可选择“小游戏”“小程序”或“自定义项目”：小游戏和小程序会创建完整的平台发布模板，项目配置底部提供“一键提交审核”；Main 自动预检、安全打包并提交，构建通过后进入运营审核，审核通过即直接发布。自定义项目只创建工作空间，不配置默认发布方式。项目成果预览 `/deliverables` 继续保留。 @@ -17,7 +17,7 @@ Makelore 是一个面向软件与视觉创作的 AI 桌面工作台。当前版 - 共享开发浏览器：AI 编程右侧提供项目级浏览器，用户与 Agent 查看并调试同一实时页面、Console 和 Network，支持本地与公网开发地址。 - 真机预览：AI 编程项目空间可读取当前静态 Release 状态；待审版本使用短时预览地址，已发布版本使用同源 HTTPS `play_url`，供手机直接测试触控、屏幕和移动网络环境。 - 后端边界：Renderer 通过 Main 所有的 Host API 访问认证、模型、同步、更新、语音、图像与运行时能力。 -- AI 绘画：每个设计项目固定一个设计 Agent，并可包含多条互相独立的设计会话。消息、Brief、Quote 和 `turnRevision` 属于 Conversation；图片/视频生成任务与资产属于 Workspace，切换或新建会话不会创建新项目，也不会隐藏项目任务。每条 Conversation 复用服务端持久 Agent Gateway Session，任务进度通过实时流推送并在统一列表中展示，断流时使用低频 REST 同步。生产环境使用 Works Square 云端 Workspace 契约，上游不可用时明确报错。 +- AI 绘画：每个设计项目固定一个设计 Agent，并可包含多条互相独立的设计会话。消息、Brief、Quote 和 `turnRevision` 属于 Conversation；图片/视频生成任务与资产属于 Workspace，切换或新建会话不会创建新项目，也不会隐藏项目任务。图片创作既支持文生图，也支持从当前项目已完成作品或本地上传中选择一张参考图继续生成；视频沿用同一单图选择器绑定首帧。每条 Conversation 复用服务端持久 Agent Gateway Session，任务进度通过实时流推送并在统一列表中展示，断流时使用低频 REST 同步。生产环境使用 Works Square 云端 Workspace 契约，上游不可用时明确报错。 - 视觉系统：单一浅色主题，品牌蓝 `#3A5578`、星火橙 `#F26A3D`、白色画布与低饱和蓝灰层级。 - 品牌资产：生产 SVG、PNG、应用图标、托盘图标与安装器视觉位于 [`resources/brand/`](resources/brand/README.md) 和 `resources/icons/`。 diff --git a/src/pages/ImageCanvas/index.tsx b/src/pages/ImageCanvas/index.tsx index a733fd3..7cb4f68 100644 --- a/src/pages/ImageCanvas/index.tsx +++ b/src/pages/ImageCanvas/index.tsx @@ -78,7 +78,7 @@ const GENERATION_CONFIRMATION_INTENTS = new Set([ '确认开始制作', '确认并开始制作', ]); -const VIDEO_FIRST_FRAME_PICKER_REPLY = '从作品列表选择图片'; +const IMAGE_SOURCE_PICKER_REPLY = '从作品列表选择图片'; function isGenerationConfirmationIntent(message: string): boolean { const normalized = message.trim().replace(/[\s，,。.!！?？、]/g, ''); @@ -259,7 +259,7 @@ function AssetPreview({ asset }: { asset: DesignAsset }) { ); } -function FirstFrameAssetOption({ +function ImageSourceAssetOption({ asset, busy, onSelect, @@ -424,10 +424,10 @@ export function ImageCanvas() { const [submittingConversationId, setSubmittingConversationId] = useState(null); const [confirmingQuoteId, setConfirmingQuoteId] = useState(null); const [actionError, setActionError] = useState(null); - const [firstFramePickerOpen, setFirstFramePickerOpen] = useState(false); - const [firstFrameBusyKey, setFirstFrameBusyKey] = useState(null); - const [firstFrameError, setFirstFrameError] = useState(null); - const [uploadedFirstFrameAsset, setUploadedFirstFrameAsset] = useState(null); + const [imageSourcePickerOpen, setImageSourcePickerOpen] = useState(false); + const [imageSourceBusyKey, setImageSourceBusyKey] = useState(null); + const [imageSourceError, setImageSourceError] = useState(null); + const [uploadedImageSourceAsset, setUploadedImageSourceAsset] = useState(null); const conversationEndRef = useRef(null); const quote = useMemo( @@ -435,20 +435,21 @@ export function ImageCanvas() { [conversation], ); const generationAvailable = bootstrap?.capabilities.generation ?? false; - const firstFrameAssets = useMemo( + const imageSourceAssets = useMemo( () => { const assets = tasks .filter((task) => task.status === 'succeeded') .flatMap((task) => task.resultAssets) .filter((asset) => asset.mediaType === 'image' && asset.mimeType === 'image/png'); - if (uploadedFirstFrameAsset - && uploadedFirstFrameAsset.workspaceId === workspace?.workspaceId) { - assets.unshift(uploadedFirstFrameAsset); + if (uploadedImageSourceAsset + && uploadedImageSourceAsset.workspaceId === workspace?.workspaceId) { + assets.unshift(uploadedImageSourceAsset); } return [...new Map(assets.map((asset) => [asset.assetId, asset])).values()]; }, - [tasks, uploadedFirstFrameAsset, workspace?.workspaceId], + [tasks, uploadedImageSourceAsset, workspace?.workspaceId], ); + const selectingImageReference = conversation?.brief.medium === 'image'; const conversationMessages = useMemo(() => { if (!conversation) return []; const messages: RenderedDesignMessage[] = [...conversation.messages]; @@ -499,8 +500,8 @@ export function ImageCanvas() { setPrompt(''); setActionError(null); setConfirmingQuoteId(null); - setFirstFramePickerOpen(false); - setFirstFrameError(null); + setImageSourcePickerOpen(false); + setImageSourceError(null); }, [taskConversationId]); useEffect(() => { @@ -569,17 +570,22 @@ export function ImageCanvas() { void handleSend(); }; - const handleFirstFrameSelect = async (asset: DesignAsset) => { - if (!workspace || !conversation || submitting || firstFrameBusyKey) return; + const handleImageSourceSelect = async (asset: DesignAsset) => { + if (!workspace || !conversation || submitting || imageSourceBusyKey) return; const requestedWorkspaceId = workspace.workspaceId; const requestedConversationId = conversation.conversationId; - setFirstFrameBusyKey(asset.assetId); - setFirstFrameError(null); + setImageSourceBusyKey(asset.assetId); + setImageSourceError(null); setActionError(null); setSubmittingConversationId(requestedConversationId); - setFirstFramePickerOpen(false); + setImageSourcePickerOpen(false); try { - await sendMessage('使用作品图片作为视频首帧', [asset.assetId]); + await sendMessage( + selectingImageReference + ? '使用作品图片作为图生图参考图' + : '使用作品图片作为视频首帧', + [asset.assetId], + ); } catch (error) { const current = useImageWorkspaceStore.getState(); if (current.activeWorkspaceId === requestedWorkspaceId @@ -587,39 +593,44 @@ export function ImageCanvas() { setActionError(error instanceof Error ? error.message : String(error)); } } finally { - setFirstFrameBusyKey(null); + setImageSourceBusyKey(null); setSubmittingConversationId((current) => ( current === requestedConversationId ? null : current )); } }; - const handleFirstFrameUpload = async (event: ChangeEvent) => { + const handleImageSourceUpload = async (event: ChangeEvent) => { const file = event.currentTarget.files?.[0]; event.currentTarget.value = ''; - if (!workspace || !conversation || !file || submitting || firstFrameBusyKey) return; + if (!workspace || !conversation || !file || submitting || imageSourceBusyKey) return; const requestedWorkspaceId = workspace.workspaceId; const requestedConversationId = conversation.conversationId; - setFirstFrameBusyKey('upload'); - setFirstFrameError(null); + setImageSourceBusyKey('upload'); + setImageSourceError(null); setSubmittingConversationId(requestedConversationId); try { const asset = await uploadImageWorkspaceAsset(requestedWorkspaceId, file); - setUploadedFirstFrameAsset(asset); - await sendMessage('使用上传的图片作为视频首帧', [asset.assetId]); + setUploadedImageSourceAsset(asset); + await sendMessage( + selectingImageReference + ? '使用上传图片作为图生图参考图' + : '使用上传的图片作为视频首帧', + [asset.assetId], + ); const current = useImageWorkspaceStore.getState(); if (current.activeWorkspaceId === requestedWorkspaceId && current.activeConversationId === requestedConversationId) { - setFirstFramePickerOpen(false); + setImageSourcePickerOpen(false); } } catch (error) { const current = useImageWorkspaceStore.getState(); if (current.activeWorkspaceId === requestedWorkspaceId && current.activeConversationId === requestedConversationId) { - setFirstFrameError(error instanceof Error ? error.message : String(error)); + setImageSourceError(error instanceof Error ? error.message : String(error)); } } finally { - setFirstFrameBusyKey(null); + setImageSourceBusyKey(null); setSubmittingConversationId((current) => ( current === requestedConversationId ? null : current )); @@ -837,9 +848,9 @@ export function ImageCanvas() { size="sm" className="h-8 rounded-full px-3 text-xs font-semibold" onClick={() => { - if (reply === VIDEO_FIRST_FRAME_PICKER_REPLY) { - setFirstFrameError(null); - setFirstFramePickerOpen(true); + if (reply === IMAGE_SOURCE_PICKER_REPLY) { + setImageSourceError(null); + setImageSourcePickerOpen(true); return; } if (!isGenerationConfirmationIntent(reply)) { @@ -935,19 +946,23 @@ export function ImageCanvas() { { - if (!open && firstFrameBusyKey) return; - setFirstFramePickerOpen(open); - if (!open) setFirstFrameError(null); + if (!open && imageSourceBusyKey) return; + setImageSourcePickerOpen(open); + if (!open) setImageSourceError(null); }} >
- 选择视频首帧 + + {selectingImageReference ? '选择图生图参考图' : '选择视频首帧'} + - 从本项目作品中选择一张已完成图片，或上传本地图片作为视频起始画面。 + {selectingImageReference + ? '从本项目作品中选择一张已完成图片，或上传本地图片作为图生图参考。' + : '从本项目作品中选择一张已完成图片，或上传本地图片作为视频起始画面。'}
@@ -955,9 +970,9 @@ export function ImageCanvas() { type="button" size="icon" variant="ghost" - aria-label="关闭首帧选择" + aria-label={selectingImageReference ? '关闭参考图选择' : '关闭首帧选择'} className="h-10 w-10 shrink-0 rounded-xl transition-transform active:scale-[0.96]" - disabled={Boolean(firstFrameBusyKey)} + disabled={Boolean(imageSourceBusyKey)} > @@ -973,43 +988,43 @@ export function ImageCanvas() {

- {firstFrameError ? ( + {imageSourceError ? (

- {firstFrameError} + {imageSourceError}

) : null} - {firstFrameAssets.length > 0 ? ( + {imageSourceAssets.length > 0 ? (
- {firstFrameAssets.map((asset) => ( - ( + void handleFirstFrameSelect(asset)} + busy={Boolean(imageSourceBusyKey)} + onSelect={() => void handleImageSourceSelect(asset)} /> ))}
diff --git a/tests/unit/image-canvas-page.test.tsx b/tests/unit/image-canvas-page.test.tsx index 407ee84..dbf7f59 100644 --- a/tests/unit/image-canvas-page.test.tsx +++ b/tests/unit/image-canvas-page.test.tsx @@ -711,11 +711,152 @@ describe('ImageCanvas Workspace-first design experience', () => { expect(sendImageWorkspaceMessageMock).not.toHaveBeenCalled(); }); + it('keeps the historical video picker behavior when the brief medium is not decided yet', async () => { + fetchImageWorkspaceConversationMock.mockResolvedValueOnce({ + ...conversationFixture(), + phase: 'shaping', + brief: { + version: 2, + status: 'draft', + medium: null, + summary: '', + ready: false, + missingDecision: '请选择首帧来源', + }, + messages: [ + ...conversationFixture().messages, + { + id: 'message-undecided-source', + role: 'assistant', + kind: 'choice', + text: '请选择视频首帧来源。', + quickReplies: ['从作品列表选择图片'], + generationQuote: null, + turnRevision: 2, + createdAt: '2026-08-12T09:00:00Z', + }, + ], + turnRevision: 2, + } satisfies DesignConversation); + render(); + + fireEvent.click(await screen.findByRole('button', { name: '从作品列表选择图片' })); + const picker = screen.getByRole('dialog', { name: '选择视频首帧' }); + const selectButton = within(picker).getByRole('button', { name: '选择此图片' }); + await waitFor(() => expect(selectButton).toBeEnabled()); + fireEvent.click(selectButton); + + await waitFor(() => expect(sendImageWorkspaceMessageMock).toHaveBeenCalledWith( + 'workspace-cloud', + 'conversation-cloud', + 2, + '使用作品图片作为视频首帧', + expect.stringMatching(/^turn-/), + ['asset-one'], + )); + expect(screen.queryByRole('dialog', { name: '选择视频首帧' })) + .not.toBeInTheDocument(); + }); + + it('opens the image reference picker and submits a successful work Asset id', async () => { + fetchImageWorkspaceConversationMock.mockResolvedValueOnce({ + ...conversationFixture(), + phase: 'shaping', + brief: { + version: 2, + status: 'draft', + medium: 'image', + summary: '参考海洋公益海报生成新图', + ready: false, + missingDecision: '请选择参考图来源', + }, + messages: [ + ...conversationFixture().messages, + { + id: 'message-image-source', + role: 'assistant', + kind: 'choice', + text: '请选择图生图参考图。', + quickReplies: ['从作品列表选择图片'], + generationQuote: null, + turnRevision: 2, + createdAt: '2026-08-12T09:00:00Z', + }, + ], + turnRevision: 2, + } satisfies DesignConversation); + render(); + + fireEvent.click(await screen.findByRole('button', { name: '从作品列表选择图片' })); + const picker = screen.getByRole('dialog', { name: '选择图生图参考图' }); + expect(within(picker).getByText(/作为图生图参考/)).toBeInTheDocument(); + const selectButton = within(picker).getByRole('button', { name: '选择此图片' }); + await waitFor(() => expect(selectButton).toBeEnabled()); + fireEvent.click(selectButton); + + await waitFor(() => expect(sendImageWorkspaceMessageMock).toHaveBeenCalledWith( + 'workspace-cloud', + 'conversation-cloud', + 2, + '使用作品图片作为图生图参考图', + expect.stringMatching(/^turn-/), + ['asset-one'], + )); + expect(screen.queryByRole('dialog', { name: '选择图生图参考图' })) + .not.toBeInTheDocument(); + }); + + it('uploads a local image reference and submits its Asset id to the Agent', async () => { + fetchImageWorkspaceConversationMock.mockResolvedValueOnce({ + ...conversationFixture(), + messages: [ + ...conversationFixture().messages, + { + id: 'message-image-source', + role: 'assistant', + kind: 'choice', + text: '请选择图生图参考图。', + quickReplies: ['从作品列表选择图片'], + generationQuote: null, + turnRevision: 2, + createdAt: '2026-08-12T09:00:00Z', + }, + ], + turnRevision: 2, + } satisfies DesignConversation); + render(); + fireEvent.click(await screen.findByRole('button', { name: '从作品列表选择图片' })); + const picker = screen.getByRole('dialog', { name: '选择图生图参考图' }); + const file = new File(['image-bytes'], 'reference.png', { type: 'image/png' }); + + fireEvent.change(within(picker).getByLabelText('上传本地图片'), { + target: { files: [file] }, + }); + + await waitFor(() => expect(uploadImageWorkspaceAssetMock) + .toHaveBeenCalledWith('workspace-cloud', file)); + await waitFor(() => expect(sendImageWorkspaceMessageMock).toHaveBeenCalledWith( + 'workspace-cloud', + 'conversation-cloud', + 2, + '使用上传图片作为图生图参考图', + expect.stringMatching(/^turn-/), + ['asset-uploaded'], + )); + await waitFor(() => expect( + screen.queryByRole('dialog', { name: '选择图生图参考图' }), + ).not.toBeInTheDocument()); + }); + it('submits a successful work Asset id when it is selected as the video first frame', async () => { const pendingSelection = deferred(); sendImageWorkspaceMessageMock.mockReturnValueOnce(pendingSelection.promise); fetchImageWorkspaceConversationMock.mockResolvedValueOnce({ ...conversationFixture(), + brief: { + ...conversationFixture().brief, + medium: 'video', + }, turnRevision: 2, messages: [ ...conversationFixture().messages, @@ -759,6 +900,10 @@ describe('ImageCanvas Workspace-first design experience', () => { it('uploads a local first-frame image and submits its Asset id to the Agent', async () => { fetchImageWorkspaceConversationMock.mockResolvedValueOnce({ ...conversationFixture(), + brief: { + ...conversationFixture().brief, + medium: 'video', + }, messages: [ ...conversationFixture().messages, { @@ -793,6 +938,9 @@ describe('ImageCanvas Workspace-first design experience', () => { expect.stringMatching(/^turn-/), ['asset-uploaded'], )); + await waitFor(() => expect( + screen.queryByRole('dialog', { name: '选择视频首帧' }), + ).not.toBeInTheDocument()); }); it('renders policy-blocked generation tasks with a user-facing design message', async () => {