From 8820e82530dcaee77d8398dce3ef34ef7a7c40b5 Mon Sep 17 00:00:00 2001 From: brother7 <7brother7@gmail.com> Date: Mon, 24 Aug 2026 13:44:39 +0800 Subject: [PATCH] test: add PI release proof harness --- .../20260824-pi-release-proof-3e725ac7.md | 114 ++++++ README.md | 4 +- docs/pi-runtime-release-runbook.md | 66 ++++ package.json | 3 + scripts/lib/pi-product-artifact.mjs | 340 ++++++++++++++++++ scripts/probe-pi-packaged-runtime.mjs | 2 +- scripts/probe-pi-runtime.mjs | 18 + scripts/run-electron-vitest.mjs | 4 +- scripts/run-pi-release-performance.mjs | 269 ++++++++++++++ scripts/run-pi-subagent-packaged-smoke.mjs | 61 +++- scripts/smoke-pi-real.mjs | 122 +++++++ scripts/verify-pi-product-artifact.mjs | 97 +++++ tests/e2e/pi-coding-first-chat.spec.ts | 94 ++++- tests/unit/coding-chat-pressure.test.tsx | 19 +- tests/unit/pi-product-artifact.test.ts | 100 ++++++ tests/unit/pi-release-docs.test.ts | 31 ++ .../pi-release-performance-fragments.test.ts | 84 +++++ tests/unit/pi-runtime-bundle.test.ts | 6 + tests/unit/pi-worker-process-real.test.ts | 17 +- 19 files changed, 1407 insertions(+), 44 deletions(-) create mode 100644 .project-docs/30-worklog/tasks/20260824-pi-release-proof-3e725ac7.md create mode 100644 docs/pi-runtime-release-runbook.md create mode 100644 scripts/lib/pi-product-artifact.mjs create mode 100644 scripts/run-pi-release-performance.mjs create mode 100644 scripts/smoke-pi-real.mjs create mode 100644 scripts/verify-pi-product-artifact.mjs create mode 100644 tests/unit/pi-product-artifact.test.ts create mode 100644 tests/unit/pi-release-docs.test.ts create mode 100644 tests/unit/pi-release-performance-fragments.test.ts diff --git a/.project-docs/30-worklog/tasks/20260824-pi-release-proof-3e725ac7.md b/.project-docs/30-worklog/tasks/20260824-pi-release-proof-3e725ac7.md new file mode 100644 index 0000000..10f480f --- /dev/null +++ b/.project-docs/30-worklog/tasks/20260824-pi-release-proof-3e725ac7.md @@ -0,0 +1,114 @@ +# Task: Implement PI-150 packaging, E2E, and performance proof + +## Identity + +- Task ID: 20260824-pi-release-proof-3e725ac7 +- Mode: Feature +- Branch: codex/20260824-pi-release-proof-3e725ac7-pi-release-proof +- Worktree: D:\Datas\OthersProjects\makelore-pi-release-proof-3e725ac7 +- Base commit: 977445ba450f4ad32b6e6db2caf048517513ab39 +- Owner: codex-root +- Status: In Progress + +## Scope + +- Implement and verify `PI-150 — Packaging, E2E, and performance release proof` + from cumulative PI-140 delivery base `977445b`. +- Own final Pi production-closure staging, electron-builder wiring, artifact + verification, actual packaged Pi/provider-shaped smoke, Coding Electron E2E, + performance evidence, release note/runbook draft, and README current-state + updates. +- Produce Windows x64 and Linux x64 final-product artifact evidence available + from this host. Keep canonical `.project-docs` updates for PI-160 Integration + Gate; this feature record contains the task-scoped evidence and promotion + candidates. + +## Intent And Constraints + +- Preserve Main-owned runtime/provider/credential boundaries, the exact pinned + Pi package and pnpm versions, Makelore app identity, project-scoped state, and + the single light product system. +- Final artifact checks must prove Pi version/engine/entry/resources/production + closure/`get_state`, extension and skill presence, no OpenCode production + residue, and no development-machine absolute path. +- `smoke:pi:real` means the actual Pi process seam from the final artifact using + controlled loopback/provider-shaped endpoints. Real external Provider + Accounts and real-provider two-worker turns are explicitly waived accepted + risks, never Pass; every substitute report retains + `realTurnVerified=false`. +- The user explicitly skipped macOS validation. Do not manufacture or reuse + static/cross-platform evidence for macOS x64 or arm64. Their missing final + artifact, closure/native/resource, loopback, and performance evidence remains + an explicit release blocker, so this task cannot claim cross-platform + release-ready or PI-150 Done. +- The canonical project memory at this base is stale and still describes + OpenCode. Treat the planner-owned PI Spec/ticket and current source as the + task authority; do not edit canonical memory in this feature worktree. +- The main worktree is occupied by an older integration task. Remain rooted in + this isolated worktree and do not inspect or modify peer uncommitted files. + +## Plan + +1. Audit the cumulative PI-140 tree against every PI-150 packaging, smoke, + E2E, performance, release-note, and README requirement; reuse only verified + existing seams and identify concrete missing behavior. +2. Implement focused missing harnesses/wiring/tests with behavior-level + regression coverage, then run typecheck and relevant unit tests. +3. Run the full pinned validation set and build the Windows x64 final product + artifact; run its verifier, actual packaged Pi/provider-shaped smoke, and + required performance scenarios with structured statistics. +4. Use WSL2 only if available to perform a frozen Linux x64 install, final + product build, verifier/smoke, and performance run; label any environment or + artifact limitation precisely. +5. Record reproducible commands/results, accepted Provider risk, skipped macOS + release blocker, release note/runbook, README delta, and PI-160 promotion + candidates; complete the Task Documentation Gate without overstating + release readiness. + +## Outcome + +- Implementation is complete enough for final-product qualification: the tree + now contains a final-product Pi artifact verifier, packaged Pi/provider-shaped + smoke, Spec 17.3 performance runner, expanded Electron E2E, release runbook, + and synchronized README current-state language. +- Final artifact execution and cross-platform evidence are still in progress. + PI-150 remains blocked from `Done` by the explicitly skipped macOS x64/arm64 + validation, regardless of Windows/Linux results. + +## Verification + +- `pnpm install --frozen-lockfile` with pnpm `10.33.4`: passed; 997 packages, + lockfile unchanged. +- Focused Pi verifier/performance/docs/runtime tests: 6 files passed, 14 tests + passed, 2 skipped. +- `pnpm run typecheck`: passed after implementation and release-doc updates. +- Scoped ESLint for every changed script/test: passed. +- `pnpm run build:vite`: passed after the implementation changes. +- `pnpm exec playwright test tests/e2e/pi-coding-first-chat.spec.ts + --workers=1`: passed, 2/2. The first run correctly exposed that complete tool + cards require user expansion and that the fixture used invalid worker status + `failed`; the final test follows the real expansion path and uses contract + status `error`. +- `pnpm run lint:check`: passed with 0 errors and 5 pre-existing warnings. +- `pnpm test`: passed; 178 files, 1504 tests passed, 2 skipped. +- Full Electron E2E after the production build: passed, 24/24. An earlier + attempt hit one Canvas hover timing failure, and the next attempt had a + Windows Playwright worker exit with code `3221226505`; both isolated paths + passed and the final uninterrupted full run passed. +- Final-product artifact verification is pending. + +## Follow-ups + +- Build and qualify Windows x64 and Linux x64 final product artifacts. +- Keep missing macOS x64/arm64 evidence as a release blocker; do not promote a + cross-platform release-ready conclusion. + +## Promotion Candidates + +- PI-160 should promote the current Pi runtime/Conversation state from README + and `docs/pi-runtime-release-runbook.md` into canonical project memory after + integration review. +- PI-160 should record QG-004/QG-005 as `Explicitly Waived / Accepted Risk`, + with `realTurnVerified=false`; this is not a Pass. +- PI-160 must preserve the macOS x64/arm64 missing-evidence blocker unless a + later task supplies independent final-product evidence. diff --git a/README.md b/README.md index a513f1e..bf500e3 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,8 @@ pnpm run package:linux Windows 打包脚本会先准备目标架构所需的 Pi、Python 与 uv 运行时资源,产物写入忽略的 `release/` 目录。AI 学习不再携带独立播放器产物。Windows 正式包中的 Pi、Python 和 uv 均从安装目录解析;缺少本地资源时启动或产物验证会直接失败,不会回退到系统 Python、npm 或 npx 下载。Git、项目编译器和用户选择的浏览器仍属于项目/系统工具,不属于内置 Pi 运行时。macOS 与 Linux 的双架构产物需要分别完成对应架构的 staging 与产物验证后再发布。 +Pi 正式包必须继续运行 `pnpm run verify:artifact:pi`、`pnpm run smoke:pi:real` 和 `pnpm run perf:pi:release`。这里的 `real` 表示从最终产品可执行文件启动最终 `resources/pi-runtime`,并使用受控的 Provider-shaped 回环服务验证会话、工具、中止、结算、重开、并发隔离、子 Agent 与退出;它不表示真实外部 Provider 已验证。各目标平台、证据字段、兼容边界与整版本回滚步骤见 [`docs/pi-runtime-release-runbook.md`](docs/pi-runtime-release-runbook.md)。 + ## 代码结构 | 路径 | 职责 | @@ -128,7 +130,7 @@ Windows 打包脚本会先准备目标架构所需的 Pi、Python 与 uv 运行 - Conversation 历史按需从 Main-owned Snapshot 读取。公开 SSE 只交付 Snapshot 与按 Conversation、worker generation 分组的 `patch-batch`;Renderer 整批校验连续 seq 后在一次状态事务中顺序应用,缺口或畸形批次只恢复目标 Conversation,隐藏 Conversation 的流式更新不会提交选中时间线。 - 核心时间线渲染消息、Markdown、thinking、工具、压缩、轮次边界、通知和 `subagent.v1` 单个/并行/串行子任务;默认保留最近 120 个节点的渲染窗口,可按 100 个节点加载更早内容。工具结果和浏览器附件保留在对应工具卡片内,累计输出覆盖同一块而不形成独立气泡;压缩只展示产品摘要、重试和结算状态。 - Composer 支持文字、粘贴或选择 PNG/JPEG/WebP/GIF 图片,每条消息最多 16 张、最多并行上传 4 张。图片在发送前只保留本地预览,点击发送时才经 Main-owned 有界二进制接口上传一次;Main 在落盘前核对 MIME 与最小图片签名,状态与事件只保存 attachment id,时间线按需读取二进制并创建临时 object URL,不保存重复 base64。 -- Makelore 在应用侧按 Session 独立提交、跟踪和隔离运行状态,不使用“当前对话正在回复”的全局界面锁;同一 Session 的后续消息仍按顺序排队。共享 runtime/provider 是否真正并发执行不同 Session 尚未经过自动化真实运行 smoke 验证,运行时仍可能自行串行、限流或拒绝请求。 +- Makelore 在应用侧按 Session 独立提交、跟踪和隔离运行状态,不使用“当前对话正在回复”的全局界面锁;同一 Session 的后续消息仍按顺序排队。最终产品中的 Pi 运行时会通过受控 Provider-shaped 回环 smoke 验证两个 worker 的重叠执行、状态隔离与凭证引用隔离;真实外部 Provider 的并发、限流、协议兼容和凭证隔离仍是独立风险,未执行真实 Provider 验证时不得标记为 Pass。 - 首次发送会立即生成稳定的乐观用户消息;HTTP 202 只表示本地 Agent 已接收。后续失败不会删除已接受消息,不确定交付不会自动重发;准备失败可在目标 Conversation 上手动恢复。 - 运行中的 Conversation 可把新消息作为 steer 引导当前回答或 follow-up 排到下一轮,并显示队列位置;队列只在 `agent_settled` 后释放,用户可中止当前运行。select/confirm/input/editor 交互在输入区上方回答,并明确展示取消或失效结果。 - 核心聊天页支持标题、归档、未读、恢复,以及“从这里创建新对话分支”;分支只创建新的 Conversation 历史,不表示文件回滚。右侧编程工具集中展示当前 Conversation 的 changes、项目文件预览、浏览器附件、技能、命令与脱敏运行诊断。分享、待办、全局运行时和 revert/unrevert 不属于该产品界面。 diff --git a/docs/pi-runtime-release-runbook.md b/docs/pi-runtime-release-runbook.md new file mode 100644 index 0000000..12ff06a --- /dev/null +++ b/docs/pi-runtime-release-runbook.md @@ -0,0 +1,66 @@ +# Pi Runtime 发布与回滚 + +本文定义 Makelore Code 当前 Pi Runtime 正式包的发布门槛、兼容边界和整版本回滚方式。它是发布操作契约,不是历史证据归档;每次执行产生的 JSON、日志、安装包和性能报告都保存在忽略的 `release/` 目录或外部发布系统中。 + +## 发布边界 + +- 正式包固定使用仓库锁定的 Pi 版本和生产依赖闭包。Electron Main 从安装目录中的 `resources/pi-runtime` 解析运行时、扩展、Manifest 与编码 Skills;缺失、版本漂移、开发路径泄漏或 OpenCode 命名资源残留均阻断发布。 +- Renderer 只使用 Main-owned Host API 和产品 Conversation 契约,不读取 Pi wire 类型,也不直接启动 worker、访问本地运行时地址或接触 Provider 凭证。 +- 产品保留既有 `niancode` 应用 id、协议、数据目录和项目配置兼容标识;这不代表旧 OpenCode 运行时、路由或资源仍被携带。 +- 正式产品界面不提供 share/unshare、revert/unrevert、todos 或全局运行时控制。项目分支只创建新的 Conversation 历史,不表示文件回滚。 + +## Conversation 兼容性 + +升级前已有的 OpenCode Conversation 数据继续保留,项目首次进入 Pi 会显示一次迁移说明,但 Pi 不导入、续写或改写这些旧 Conversation。升级后新建的 Pi Conversation 使用新的本地契约;旧版本不能读取或续写它们。 + +因此,旧 Conversation 的“保留”是可回滚的数据边界,不是跨运行时续写兼容。不得通过复制 Session、混用资源目录或在同一安装中切换旧/新 runtime 来绕过该边界。 + +## 必须验证的正式产物 + +发布候选必须分别提供以下最终产品证据: + +| 目标 | 最终产物 | 必须结果 | +| --- | --- | --- | +| Windows x64 | 安装器与 `win-unpacked` | 产物验证、Pi smoke、性能报告均 Pass | +| Linux x64 | 发布包与 `linux-unpacked` | 产物验证、Pi smoke、性能报告均 Pass | +| macOS x64 | 对应架构应用包 | 独立产物验证与 Pi smoke Pass | +| macOS arm64 | 对应架构应用包 | 独立产物验证与 Pi smoke Pass | + +缺少任一目标的独立证据时,结论只能是 `Blocked`,不能用其他平台、staging 目录、源码测试或用户豁免替代跨平台发布就绪结论。 + +每个平台在完成打包后,从该平台的最终应用可执行文件运行: + +```bash +pnpm run verify:artifact:pi -- --app-exe --samples 5 --report +pnpm run smoke:pi:real -- --app-exe --samples 5 --report +pnpm run perf:pi:release -- --app-exe --samples 5 --report +``` + +产物验证必须确认版本与 Node engine、生产依赖闭包、扩展、Manifest、编码 Skills、`resolve/get_state`、无 OpenCode 资源以及无构建工作区绝对路径。Smoke 必须从最终产品可执行文件启动最终 `resources/pi-runtime`,覆盖 session、prompt、tool、abort、settle、reopen、双 worker 重叠与隔离、子 Agent 和 shutdown。性能报告必须记录 p50/p95/max/样本数组、RSS、Main→Renderer IPC/提交延迟和 Git commit,并覆盖 Spec 17.3 的十个场景。 + +## Provider 验证的准确含义 + +`smoke:pi:real` 中的 `real` 指真实最终 Pi 进程与最终产品资源,不指真实外部 Provider。受控 Provider-shaped 回环服务只证明 Makelore/Pi 的协议与隔离路径按预期运行。 + +如果发布决定明确豁免真实 Provider 验证,报告必须保持 `realTurnVerified=false`,并把 QG-004/QG-005 标为 `Explicitly Waived / Accepted Risk`,不得写成 Pass。被接受的剩余风险包括真实 Provider 的协议差异、限流与并发策略、凭证隔离,以及多模态输入的供应商特定行为;报告和日志不得包含真实凭证。 + +## 发布判定 + +只有下列条件同时成立才能发布: + +1. 固定 pnpm 版本的 frozen install、typecheck、lint、unit、production build 和完整 Electron E2E 通过。 +2. 四个目标的最终产品产物验证与 Pi smoke 均通过,且报告对应同一 Git commit。 +3. Spec 17.3 场景与性能预算通过,没有源码目录、staging 目录或开发 fallback 代替最终产物证据。 +4. 真实 Provider 若未验证,已由发布负责人显式接受上述风险,并保持非 Pass 状态。 + +## 整版本回滚 + +Pi 与旧运行时之间只支持完整应用版本回滚,不支持运行时组件级回滚或混装: + +1. 停止发布和自动更新,完整备份目标用户数据目录及项目中的 `.niancode` 数据。 +2. 退出 Makelore,确认没有 Pi worker 或安装器进程仍在运行。 +3. 安装上一完整、已验证的应用版本;不得把旧 runtime 覆盖到新版本 `resources`,也不得保留新旧文件拼接后的安装目录。 +4. 旧版本只继续使用其原本可读的旧 Conversation。升级后新建的 Pi Conversation 保留在磁盘中,但对旧版本不兼容,不能读取或续写。 +5. 需要重新升级时,安装完整 Pi 版本并重新执行该版本的项目迁移与一次性说明流程;不要在不同版本间复制 Session 文件。 + +回滚完成后,应验证应用标识与用户数据目录未改变、旧 Conversation 仍可按旧版本能力读取、Pi Conversation 字节未被旧版本改写。任何需要手工编辑 Conversation 文件才能恢复的情况都应停止操作并保留备份。 diff --git a/package.json b/package.json index 4303088..840e08d 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,9 @@ "probe:pi-runtime:packaged": "node scripts/probe-pi-packaged-runtime.mjs", "probe:pi-provider-contracts": "node scripts/probe-pi-provider-contracts.mjs", "bundle:pi-runtime": "node scripts/bundle-pi-runtime.mjs", + "verify:artifact:pi": "node scripts/verify-pi-product-artifact.mjs", + "smoke:pi:real": "node scripts/smoke-pi-real.mjs", + "perf:pi:release": "node scripts/run-pi-release-performance.mjs", "lint": "eslint . --fix", "lint:check": "eslint .", "typecheck": "tsc --noEmit", diff --git a/scripts/lib/pi-product-artifact.mjs b/scripts/lib/pi-product-artifact.mjs new file mode 100644 index 0000000..dc2e469 --- /dev/null +++ b/scripts/lib/pi-product-artifact.mjs @@ -0,0 +1,340 @@ +import { spawn } from 'node:child_process'; +import { readFile, readdir, stat } from 'node:fs/promises'; +import { arch as hostArch, platform as hostPlatform } from 'node:os'; +import { + dirname, + isAbsolute, + join, + relative, + resolve, + sep, +} from 'node:path'; +import YAML from 'yaml'; + +import { + PI_RUNTIME_CLI_ENTRY, + PI_RUNTIME_MANIFEST, + PI_RUNTIME_PACKAGE, + PI_RUNTIME_VERSION, +} from './pi-runtime-bundle.mjs'; +import { + inspectPackagedClosure, + packagedResourcesDirectory, +} from '../probe-pi-packaged-runtime.mjs'; + +const PRODUCT_NAME = 'Makelore'; +const EXTENSION_MARKER = 'makelore-runtime-v3.mjs'; + +async function pathExists(path) { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +function portable(path) { + return path.split(sep).join('/'); +} + +function normalizeLockedVersion(value) { + return typeof value === 'string' ? value.split('(', 1)[0] : null; +} + +function runCommand(executable, args, options = {}) { + return new Promise((resolvePromise, reject) => { + const child = spawn(executable, args, { + cwd: options.cwd, + env: options.env ?? process.env, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); + child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) resolvePromise({ stdout, stderr }); + else reject(new Error( + `${executable} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}: ${stderr || stdout}`, + )); + }); + }); +} + +export function defaultProductExecutable(projectRoot, currentPlatform = hostPlatform()) { + const root = resolve(projectRoot); + if (currentPlatform === 'win32') { + return join(root, 'release', 'win-unpacked', `${PRODUCT_NAME}.exe`); + } + if (currentPlatform === 'darwin') { + return join(root, 'release', 'mac', `${PRODUCT_NAME}.app`, 'Contents', 'MacOS', PRODUCT_NAME); + } + if (currentPlatform === 'linux') { + return join(root, 'release', 'linux-unpacked', PRODUCT_NAME.toLowerCase()); + } + throw new Error(`Unsupported product artifact platform: ${currentPlatform}`); +} + +export function assertNodeEngineCompatible(engine, nodeVersion) { + const match = /^>=(\d+)\.(\d+)\.(\d+)$/.exec(engine ?? ''); + if (!match) throw new Error(`Unsupported Pi Node engine expression: ${engine ?? 'missing'}`); + const required = match.slice(1).map(Number); + const actualMatch = /^(\d+)\.(\d+)\.(\d+)/.exec(nodeVersion ?? ''); + if (!actualMatch) throw new Error(`Invalid packaged Node version: ${nodeVersion ?? 'missing'}`); + const actual = actualMatch.slice(1).map(Number); + for (let index = 0; index < required.length; index += 1) { + if (actual[index] > required[index]) return; + if (actual[index] < required[index]) { + throw new Error(`Packaged Node ${nodeVersion} does not satisfy Pi engine ${engine}`); + } + } +} + +export function validatePiArtifactMetadata({ + rootPackage, + lockfile, + packagedPackage, + runtimePackage, + manifest, + runtimePlatform, +}) { + const rootDependency = rootPackage.dependencies?.[PI_RUNTIME_PACKAGE]; + const packagedDependency = packagedPackage.dependencies?.[PI_RUNTIME_PACKAGE]; + const lockDependency = lockfile.importers?.['.']?.dependencies?.[PI_RUNTIME_PACKAGE]; + const lockedVersion = normalizeLockedVersion(lockDependency?.version); + const versions = { + expected: PI_RUNTIME_VERSION, + rootDependency, + packagedDependency, + lockSpecifier: lockDependency?.specifier ?? null, + lockedVersion, + runtimePackage: runtimePackage.version, + manifest: manifest.runtime?.version ?? null, + }; + const mismatched = Object.entries(versions) + .filter(([key, value]) => key !== 'expected' && value !== PI_RUNTIME_VERSION); + if (mismatched.length > 0) { + throw new Error(`Pi artifact versions do not match: ${JSON.stringify(versions)}`); + } + if (runtimePackage.name !== PI_RUNTIME_PACKAGE) { + throw new Error(`Packaged Pi package name is ${runtimePackage.name ?? 'missing'}`); + } + if (runtimePackage.bin?.pi !== PI_RUNTIME_CLI_ENTRY) { + throw new Error(`Packaged Pi CLI entry is ${runtimePackage.bin?.pi ?? 'missing'}`); + } + if (manifest.runtime?.packageName !== PI_RUNTIME_PACKAGE + || manifest.runtime?.cliEntry !== PI_RUNTIME_CLI_ENTRY) { + throw new Error(`Packaged Pi manifest identity is invalid: ${JSON.stringify(manifest.runtime)}`); + } + if (manifest.target?.platform !== runtimePlatform.platform + || manifest.target?.arch !== runtimePlatform.arch) { + throw new Error( + `Packaged Pi target ${manifest.target?.platform ?? 'missing'}-${manifest.target?.arch ?? 'missing'} ` + + `does not match executable ${runtimePlatform.platform}-${runtimePlatform.arch}`, + ); + } + assertNodeEngineCompatible(manifest.runtime?.nodeEngine, runtimePlatform.node); + return versions; +} + +export function collectAbsoluteManifestValues(value, at = '$', results = []) { + if (typeof value === 'string') { + if (isAbsolute(value) || /^[A-Za-z]:[\\/]/.test(value)) results.push({ at, value }); + return results; + } + if (Array.isArray(value)) { + value.forEach((entry, index) => collectAbsoluteManifestValues(entry, `${at}[${index}]`, results)); + return results; + } + if (value && typeof value === 'object') { + for (const [key, entry] of Object.entries(value)) { + collectAbsoluteManifestValues(entry, `${at}.${key}`, results); + } + } + return results; +} + +export async function collectForbiddenResourcePaths(root, pattern = /opencode/i) { + const matches = []; + const visit = async (directory) => { + if (!await pathExists(directory)) return; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (pattern.test(entry.name)) matches.push(portable(relative(root, path))); + if (entry.isDirectory()) await visit(path); + } + }; + await visit(root); + return matches.sort(); +} + +async function filesContainingNeedles(root, needles) { + const matches = []; + const visit = async (path) => { + const details = await stat(path); + if (details.isDirectory()) { + for (const entry of await readdir(path, { withFileTypes: true })) { + await visit(join(path, entry.name)); + } + return; + } + const contents = await readFile(path); + const found = needles.filter((needle) => needle.length > 0 && contents.includes(needle)); + if (found.length > 0) matches.push({ path, needles: found.map((value) => value.toString()) }); + }; + await visit(root); + return matches; +} + +async function inspectProductRuntime(executable, resourcesDirectory) { + const script = String.raw` + const { createRequire } = require('node:module'); + const path = require('node:path'); + const resources = process.env.MAKELORE_PI_PRODUCT_RESOURCES; + const appRequire = createRequire(path.join(resources, 'app.asar', 'package.json')); + const packagedPackage = appRequire('./package.json'); + process.stdout.write(JSON.stringify({ + platform: process.platform, + arch: process.arch, + node: process.versions.node, + electron: process.versions.electron, + packagedPackage, + })); + `; + const { stdout } = await runCommand(executable, ['-e', script], { + env: { + ...process.env, + ELECTRON_RUN_AS_NODE: '1', + MAKELORE_PI_PRODUCT_RESOURCES: resourcesDirectory, + }, + }); + return JSON.parse(stdout.trim()); +} + +async function packagedSkillIds(resourcesDirectory) { + const root = join(resourcesDirectory, 'resources', 'coding-skills'); + if (!await pathExists(root)) throw new Error(`Packaged coding skills are missing: ${root}`); + const entries = await readdir(root, { withFileTypes: true }); + const ids = []; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (!await pathExists(join(root, entry.name, 'SKILL.md'))) { + throw new Error(`Packaged coding skill has no SKILL.md: ${entry.name}`); + } + ids.push(entry.name); + } + return ids.sort(); +} + +async function sourceSkillIds(projectRoot) { + const root = join(projectRoot, 'resources', 'coding-skills'); + const entries = await readdir(root, { withFileTypes: true }); + return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort(); +} + +export async function verifyPiProductArtifact({ projectRoot, executable }) { + const root = resolve(projectRoot); + const appExecutable = resolve(executable ?? defaultProductExecutable(root)); + if (!await pathExists(appExecutable)) throw new Error(`Product executable is missing: ${appExecutable}`); + const resourcesDirectory = packagedResourcesDirectory(appExecutable); + const runtimeRoot = join(resourcesDirectory, 'pi-runtime'); + const appAsar = join(resourcesDirectory, 'app.asar'); + for (const required of [runtimeRoot, appAsar]) { + if (!await pathExists(required)) throw new Error(`Product artifact resource is missing: ${required}`); + } + + const [rootPackageSource, lockfileSource, runtimePackageSource, manifestSource] = await Promise.all([ + readFile(join(root, 'package.json'), 'utf8'), + readFile(join(root, 'pnpm-lock.yaml'), 'utf8'), + readFile(join(runtimeRoot, 'package.json'), 'utf8'), + readFile(join(runtimeRoot, PI_RUNTIME_MANIFEST), 'utf8'), + ]); + const rootPackage = JSON.parse(rootPackageSource); + const lockfile = YAML.parse(lockfileSource); + const runtimePackage = JSON.parse(runtimePackageSource); + const manifest = JSON.parse(manifestSource); + const runtimePlatform = await inspectProductRuntime(appExecutable, resourcesDirectory); + const versions = validatePiArtifactMetadata({ + rootPackage, + lockfile, + packagedPackage: runtimePlatform.packagedPackage, + runtimePackage, + manifest, + runtimePlatform, + }); + + const absoluteManifestValues = collectAbsoluteManifestValues(manifest); + if (absoluteManifestValues.length > 0) { + throw new Error(`Pi runtime manifest contains absolute paths: ${JSON.stringify(absoluteManifestValues)}`); + } + const forbiddenResourcePaths = await collectForbiddenResourcePaths(resourcesDirectory); + if (forbiddenResourcePaths.length > 0) { + throw new Error(`Product resources contain OpenCode paths: ${forbiddenResourcePaths.join(', ')}`); + } + + const expectedSkills = await sourceSkillIds(root); + const actualSkills = await packagedSkillIds(resourcesDirectory); + if (JSON.stringify(actualSkills) !== JSON.stringify(expectedSkills)) { + throw new Error(`Packaged coding skills differ: expected ${expectedSkills}, got ${actualSkills}`); + } + const appAsarContents = await readFile(appAsar); + if (!appAsarContents.includes(Buffer.from(EXTENSION_MARKER))) { + throw new Error(`Packaged app.asar does not contain ${EXTENSION_MARKER}`); + } + + const sourceNeedles = [ + Buffer.from(root), + Buffer.from(portable(root)), + ]; + const devPathResidue = [ + ...await filesContainingNeedles(appAsar, sourceNeedles), + ...await filesContainingNeedles(runtimeRoot, sourceNeedles), + ]; + if (devPathResidue.length > 0) { + throw new Error(`Product artifact contains development path residue: ${JSON.stringify(devPathResidue)}`); + } + + const packagedClosure = await inspectPackagedClosure( + appExecutable, + resourcesDirectory, + manifest.runtimeAssets, + ); + return { + schemaVersion: 1, + artifact: { + executable: appExecutable, + resourcesDirectory, + appAsar, + runtimeRoot, + cliPath: join(runtimeRoot, ...PI_RUNTIME_CLI_ENTRY.split('/')), + }, + platform: { + platform: runtimePlatform.platform, + arch: runtimePlatform.arch, + electron: runtimePlatform.electron, + node: runtimePlatform.node, + }, + versions, + manifest: { + target: manifest.target, + packageCount: manifest.productionPackages.length, + assetCount: manifest.runtimeAssets.length, + nodeEngine: manifest.runtime.nodeEngine, + }, + packagedClosure, + extension: { marker: EXTENSION_MARKER, packaged: true }, + skills: actualSkills, + openCodeResourcePaths: [], + developmentPathResidue: [], + result: 'pass', + }; +} + +export const PI_PRODUCT_ARTIFACT_DEFAULTS = Object.freeze({ + platform: hostPlatform(), + arch: hostArch(), + extensionMarker: EXTENSION_MARKER, +}); diff --git a/scripts/probe-pi-packaged-runtime.mjs b/scripts/probe-pi-packaged-runtime.mjs index 4af6e93..bed6852 100644 --- a/scripts/probe-pi-packaged-runtime.mjs +++ b/scripts/probe-pi-packaged-runtime.mjs @@ -90,7 +90,7 @@ function runCommand(executable, args, options = {}) { }); } -async function inspectPackagedClosure(executable, resourcesDirectory, assets) { +export async function inspectPackagedClosure(executable, resourcesDirectory, assets) { const script = String.raw` const fs = require('node:fs'); const path = require('node:path'); diff --git a/scripts/probe-pi-runtime.mjs b/scripts/probe-pi-runtime.mjs index 08b6da6..ec2ae8f 100644 --- a/scripts/probe-pi-runtime.mjs +++ b/scripts/probe-pi-runtime.mjs @@ -857,9 +857,16 @@ async function promptAndSettle(worker, message, images, timeoutMs) { (event) => worker.events.indexOf(event) >= eventStartIndex && event.type === 'agent_settled', timeoutMs, ); + const firstProviderEvent = worker.events + .slice(eventStartIndex) + .find((event) => event !== agentStart && event.receivedAt >= agentStart.receivedAt); const messagesResponse = await worker.request({ type: 'get_messages' }); return { acceptedMs: accepted.receivedAt - sentAt, + agentStartMs: agentStart.receivedAt - accepted.receivedAt, + providerFirstEventMs: firstProviderEvent + ? firstProviderEvent.receivedAt - agentStart.receivedAt + : settled.receivedAt - agentStart.receivedAt, startedAt: agentStart.receivedAt, settledAt: settled.receivedAt, stopReason: lastAssistantStopReason(messagesResponse), @@ -944,7 +951,18 @@ export async function runProviderQualification(runtime, scratchRoot, fixturePath imageInput: Boolean(imagePath), distinctSessionIds: true, overlapMs: Math.round(overlapMs), + promptAcceptedSamplesMs: [leftTurn.acceptedMs, rightTurn.acceptedMs].map(Math.round), promptAcceptedMs: summarizeMeasurements([leftTurn.acceptedMs, rightTurn.acceptedMs]), + agentStartSamplesMs: [leftTurn.agentStartMs, rightTurn.agentStartMs].map(Math.round), + agentStartMs: summarizeMeasurements([leftTurn.agentStartMs, rightTurn.agentStartMs]), + providerFirstEventSamplesMs: [ + leftTurn.providerFirstEventMs, + rightTurn.providerFirstEventMs, + ].map(Math.round), + providerFirstEventMs: summarizeMeasurements([ + leftTurn.providerFirstEventMs, + rightTurn.providerFirstEventMs, + ]), stopReasons: [leftTurn.stopReason, rightTurn.stopReason], abortIsolation: { abortedStopReason, diff --git a/scripts/run-electron-vitest.mjs b/scripts/run-electron-vitest.mjs index 1c911a8..33b3b76 100644 --- a/scripts/run-electron-vitest.mjs +++ b/scripts/run-electron-vitest.mjs @@ -14,7 +14,9 @@ const vitestArgs = rawArgs.filter( (arg) => arg !== '--verify-release-runtime' && arg !== '--', ); -const electronExecutable = require('electron'); +const electronExecutable = process.env.MAKELORE_ELECTRON_EXECUTABLE + ? resolve(process.env.MAKELORE_ELECTRON_EXECUTABLE) + : require('electron'); const electronPackagePath = require.resolve('electron/package.json'); const electronPackage = JSON.parse(readFileSync(electronPackagePath, 'utf8')); const vitestPackagePath = require.resolve('vitest/package.json'); diff --git a/scripts/run-pi-release-performance.mjs b/scripts/run-pi-release-performance.mjs new file mode 100644 index 0000000..3fcd5ca --- /dev/null +++ b/scripts/run-pi-release-performance.mjs @@ -0,0 +1,269 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { arch, platform, release, tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs'; +import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs'; +import { runProbe, summarizeMeasurements } from './probe-pi-runtime.mjs'; +import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs'; + +function runCommand(executable, args, options = {}) { + return new Promise((resolvePromise, reject) => { + const child = spawn(executable, args, { + cwd: options.cwd, + env: options.env ?? process.env, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); + child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) resolvePromise({ stdout, stderr }); + else reject(new Error( + `${executable} ${args.join(' ')} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}:\n${stderr || stdout}`, + )); + }); + }); +} + +async function readJson(path) { + return JSON.parse(await readFile(path, 'utf8')); +} + +async function readJsonLines(path) { + return (await readFile(path, 'utf8')) + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + +function everyBudgetPass(budgets) { + return Object.values(budgets).every((value) => value === true); +} + +async function gitEvidence(projectRoot) { + const [{ stdout: commit }, { stdout: status }] = await Promise.all([ + runCommand('git', ['rev-parse', 'HEAD'], { cwd: projectRoot }), + runCommand('git', ['status', '--porcelain'], { cwd: projectRoot }), + ]); + return { commit: commit.trim(), dirty: Boolean(status.trim()) }; +} + +async function runLocalPerformanceFragments(projectRoot, scratchRoot, samples) { + const vitestCli = resolve(projectRoot, 'node_modules', 'vitest', 'vitest.mjs'); + const metadataPath = join(scratchRoot, 'metadata.json'); + const pressurePath = join(scratchRoot, 'pressure.json'); + const focusedFiles = [ + 'tests/unit/pi-release-performance-fragments.test.ts', + 'tests/unit/coding-chat-pressure.test.tsx', + ]; + const focused = await runCommand(process.execPath, [ + vitestCli, + 'run', + ...focusedFiles, + '--maxWorkers=1', + ], { + cwd: projectRoot, + env: { + ...process.env, + MAKELORE_PI_PERF_SAMPLES: String(samples), + MAKELORE_PI_PERF_METADATA_FRAGMENT: metadataPath, + MAKELORE_PI_PERF_PRESSURE_FRAGMENT: pressurePath, + }, + }); + + const scenarioFiles = [ + 'tests/unit/pi-worker-pool.test.ts', + 'tests/unit/pi-subagent.test.ts', + 'tests/unit/pi-write-lease.test.ts', + 'tests/unit/pi-session-projector.test.ts', + 'tests/unit/pi-event-projector.test.ts', + 'tests/unit/coding-attachments-routes.test.ts', + ]; + const scenarios = await runCommand(process.execPath, [ + vitestCli, + 'run', + ...scenarioFiles, + '--maxWorkers=1', + ], { cwd: projectRoot }); + return { + metadata: await readJson(metadataPath), + pressure: await readJson(pressurePath), + commands: { + focused: { files: focusedFiles, result: 'pass', outputBytes: focused.stdout.length }, + scenarios: { files: scenarioFiles, result: 'pass', outputBytes: scenarios.stdout.length }, + }, + }; +} + +async function runComposerSamples(projectRoot, scratchRoot, samples) { + const reportPath = join(scratchRoot, 'composer.jsonl'); + const playwrightCli = resolve(projectRoot, 'node_modules', '@playwright', 'test', 'cli.js'); + await runCommand(process.execPath, [ + playwrightCli, + 'test', + 'tests/e2e/pi-coding-first-chat.spec.ts', + '--grep=first PI Conversation', + `--repeat-each=${samples}`, + '--workers=1', + ], { + cwd: projectRoot, + env: { ...process.env, MAKELORE_PI_PERF_COMPOSER_FRAGMENT: reportPath }, + }); + const values = (await readJsonLines(reportPath)).map(({ editableMs }) => editableMs); + if (values.length !== samples) { + throw new Error(`Expected ${samples} Composer samples, got ${values.length}`); + } + return summarizeMeasurements(values); +} + +export async function runPiReleasePerformance(options) { + const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-release-performance-')); + try { + const artifact = await verifyPiProductArtifact(options); + const common = { + samples: options.samples, + timeoutMs: options.timeoutMs, + stage: false, + keepStage: false, + reportPath: undefined, + providerFixturePath: undefined, + imagePath: undefined, + electronExecutablePath: artifact.artifact.executable, + cliPath: artifact.artifact.cliPath, + artifactLabel: 'final-makelore-product-artifact', + }; + const runtime = await runProbe(common, options.projectRoot); + if (runtime.result === 'fail') throw new Error('Packaged runtime performance probe failed'); + const providerContracts = await runLocalProviderContracts(common, options.projectRoot); + const fragments = await runLocalPerformanceFragments( + options.projectRoot, + scratchRoot, + options.samples, + ); + const composerInteractiveMs = await runComposerSamples( + options.projectRoot, + scratchRoot, + options.samples, + ); + const promptAcceptedSamples = providerContracts.protocols.flatMap( + ({ qualification }) => qualification.promptAcceptedSamplesMs, + ); + const agentStartSamples = providerContracts.protocols.flatMap( + ({ qualification }) => qualification.agentStartSamplesMs, + ); + const providerFirstEventSamples = providerContracts.protocols.flatMap( + ({ qualification }) => qualification.providerFirstEventSamplesMs, + ); + const promptAcceptedMs = summarizeMeasurements(promptAcceptedSamples); + const agentStartMs = summarizeMeasurements(agentStartSamples); + const providerFirstEventMs = summarizeMeasurements(providerFirstEventSamples); + const rendererFirstCommitMs = fragments.pressure.mainToReactMs; + const budgets = { + projectMetadata: fragments.metadata.projectMetadataMs.p95 <= 1_000, + agentMetadata: fragments.metadata.agentMetadataMs.p95 <= 500, + conversationMetadata: fragments.metadata.conversationMetadataMs.p95 <= 500, + composerInteractive: composerInteractiveMs.p95 <= 500, + warmRpcReady: runtime.performance.warmReadyMs.p95 <= 1_500, + coldRpcReady: runtime.performance.coldReadyMs.p95 <= 3_000, + warmPromptAccepted: promptAcceptedMs.p95 <= 250, + coldPromptAccepted: promptAcceptedMs.p95 <= 3_000, + rendererFirstCommit: rendererFirstCommitMs.p95 <= 50, + gracefulShutdown: runtime.performance.exitMs.p95 <= 3_000, + }; + if (!everyBudgetPass(budgets)) { + throw new Error(`PI-150 performance budget failed: ${JSON.stringify(budgets)}`); + } + const git = await gitEvidence(options.projectRoot); + const scenarios = [ + { id: 1, name: 'fresh userData metadata and Composer', evidence: ['metadata fragments', 'Electron Composer samples'], result: 'pass' }, + { id: 2, name: 'first prompt milestone split', evidence: ['packaged rpc.ready', 'provider-shaped accepted/agent-start/first-event', 'Renderer commit'], result: 'pass' }, + { id: 3, name: 'warm Conversation restore', evidence: ['packaged session stop/reopen/get_entries', 'warm rpc.ready samples'], result: 'pass' }, + { id: 4, name: 'two projects provider-shaped overlap and abort isolation', evidence: providerContracts.protocols.map(({ protocol }) => protocol), result: 'pass' }, + { id: 5, name: 'same-project read-only concurrency', evidence: ['pi-worker-pool.test.ts', 'pi-subagent.test.ts'], result: 'pass' }, + { id: 6, name: 'same-project mutation lease serialization and cancellation', evidence: ['pi-write-lease.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' }, + { id: 7, name: 'four-child subagent pressure and global cap', evidence: ['pi-subagent.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' }, + { id: 8, name: '100 KB mixed blocks and batch recovery', evidence: ['coding-chat-pressure.test.tsx', 'pi-session-projector.test.ts', 'pi-event-projector.test.ts'], result: 'pass' }, + { id: 9, name: 'large image attachment references', evidence: ['repeated Electron large-image attachment E2E', 'four-protocol packaged image requests'], result: 'pass' }, + { id: 10, name: 'worker crash, stream recovery, and shutdown cleanup', evidence: ['pi-worker-pool.test.ts', 'pi-session-projector.test.ts', 'packaged exit samples'], result: 'pass' }, + ]; + const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + git, + platform: { platform: platform(), arch: arch(), release: release(), packaged: true }, + samples: options.samples, + localOverhead: { + ...fragments.metadata, + composerInteractiveMs, + coldRpcReadyMs: runtime.performance.coldReadyMs, + warmRpcReadyMs: runtime.performance.warmReadyMs, + promptAcceptedMs, + agentStartMs, + rendererFirstCommitMs, + exitMs: runtime.performance.exitMs, + }, + controlledProviderShaped: { + firstEventMs: providerFirstEventMs, + protocols: providerContracts.protocols.map(({ protocol, qualification }) => ({ + protocol, + overlapMs: qualification.overlapMs, + abortIsolation: qualification.abortIsolation.passed, + imageRequests: qualification.imageInput, + })), + realTurnVerified: false, + realProviderDecision: 'explicitly-waived-accepted-risk', + }, + rssKb: runtime.performance.rssKb, + ipc: { + runtimePatchItems: fragments.pressure.runtimePatchItems, + patchBatches: fragments.pressure.patchBatches, + sseFrames: fragments.pressure.sseFrames, + wireBytes: fragments.pressure.wireBytes, + rendererTransactions: fragments.pressure.rendererTransactions, + reactCommits: fragments.pressure.reactCommits, + }, + scenarios, + commands: fragments.commands, + budgets, + result: 'pass', + crossPlatformReleaseReady: false, + releaseBlockers: [ + 'macOS x64 and macOS arm64 PI-150 performance/final-artifact validation was skipped by explicit user direction', + ], + }; + if (options.reportPath) { + await mkdir(dirname(options.reportPath), { recursive: true }); + await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`); + } + return report; + } finally { + await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + } +} + +async function main() { + const options = parsePiArtifactVerifierArgs(process.argv.slice(2)); + if (options.help) { + process.stdout.write('Usage: node scripts/run-pi-release-performance.mjs [--app-exe path] [--samples count] [--timeout-ms ms] [--report path]\n'); + return; + } + const report = await runPiReleasePerformance(options); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} + +const isMain = process.argv[1] + && pathToFileURL(resolve(process.argv[1])).href === import.meta.url; +if (isMain) { + main().catch((error) => { + process.stderr.write(`${error.stack ?? error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/run-pi-subagent-packaged-smoke.mjs b/scripts/run-pi-subagent-packaged-smoke.mjs index cc227fb..cc33faa 100644 --- a/scripts/run-pi-subagent-packaged-smoke.mjs +++ b/scripts/run-pi-subagent-packaged-smoke.mjs @@ -6,7 +6,24 @@ import { join, resolve } from 'node:path'; import { bundlePiRuntime } from './bundle-pi-runtime.mjs'; import { defaultPiBundleTarget } from './lib/pi-runtime-bundle.mjs'; -function runVitest(runtimeRoot) { +function parseArgs(argv) { + const options = { runtimeRoot: undefined, electronExecutable: undefined }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${argument} requires a value`); + if (argument === '--runtime-root') options.runtimeRoot = resolve(value); + else if (argument === '--electron-executable') options.electronExecutable = resolve(value); + else throw new Error(`Unknown argument: ${argument}`); + index += 1; + } + if (Boolean(options.runtimeRoot) !== Boolean(options.electronExecutable)) { + throw new Error('--runtime-root and --electron-executable must be provided together'); + } + return options; +} + +function runVitest(runtimeRoot, electronExecutable) { return new Promise((resolvePromise, reject) => { const child = spawn(process.execPath, [ resolve('node_modules/vitest/vitest.mjs'), @@ -14,7 +31,11 @@ function runVitest(runtimeRoot) { 'tests/unit/pi-worker-process-real.test.ts', ], { cwd: process.cwd(), - env: { ...process.env, MAKELORE_PI_STAGED_RUNTIME_ROOT: runtimeRoot }, + env: { + ...process.env, + MAKELORE_PI_STAGED_RUNTIME_ROOT: runtimeRoot, + ...(electronExecutable ? { MAKELORE_PI_ELECTRON_EXECUTABLE: electronExecutable } : {}), + }, stdio: 'inherit', windowsHide: true, }); @@ -30,13 +51,20 @@ function runVitest(runtimeRoot) { }); } -function runElectronProductTools() { +function runElectronProductTools(electronExecutable) { return new Promise((resolvePromise, reject) => { const child = spawn(process.execPath, [ resolve('scripts/run-electron-vitest.mjs'), 'tests/unit/pi-extension-bundle.test.ts', ], { cwd: process.cwd(), + env: { + ...process.env, + ...(electronExecutable ? { + MAKELORE_ELECTRON_EXECUTABLE: electronExecutable, + MAKELORE_PI_ELECTRON_EXECUTABLE: electronExecutable, + } : {}), + }, stdio: 'inherit', windowsHide: true, }); @@ -52,15 +80,24 @@ function runElectronProductTools() { }); } -const outputRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-subagent-package-')); +const options = parseArgs(process.argv.slice(2)); +const outputRoot = options.runtimeRoot + ? null + : await mkdtemp(join(tmpdir(), 'makelore-pi-subagent-package-')); try { - const [bundle] = await bundlePiRuntime({ - outputRoot, - targets: [defaultPiBundleTarget()], - }); - if (!bundle) throw new Error('Pi runtime bundler returned no staged runtime'); - await runVitest(bundle.destination); - await runElectronProductTools(); + let runtimeRoot = options.runtimeRoot; + if (!runtimeRoot) { + const [bundle] = await bundlePiRuntime({ + outputRoot, + targets: [defaultPiBundleTarget()], + }); + if (!bundle) throw new Error('Pi runtime bundler returned no staged runtime'); + runtimeRoot = bundle.destination; + } + await runVitest(runtimeRoot, options.electronExecutable); + await runElectronProductTools(options.electronExecutable); } finally { - await rm(outputRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + if (outputRoot) { + await rm(outputRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + } } diff --git a/scripts/smoke-pi-real.mjs b/scripts/smoke-pi-real.mjs new file mode 100644 index 0000000..7539713 --- /dev/null +++ b/scripts/smoke-pi-real.mjs @@ -0,0 +1,122 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { arch, platform } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs'; +import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs'; +import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs'; +import { runProbe } from './probe-pi-runtime.mjs'; + +function runExtensionSmoke(projectRoot, artifact) { + return new Promise((resolvePromise, reject) => { + const child = spawn(process.execPath, [ + resolve(projectRoot, 'scripts', 'run-pi-subagent-packaged-smoke.mjs'), + '--runtime-root', artifact.artifact.runtimeRoot, + '--electron-executable', artifact.artifact.executable, + ], { + cwd: projectRoot, + stdio: 'inherit', + windowsHide: true, + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) resolvePromise({ result: 'pass' }); + else reject(new Error( + `Final product extension/subagent smoke failed with code ${code ?? 'null'} signal ${signal ?? 'none'}`, + )); + }); + }); +} + +export async function runRealPiSmoke(options) { + const artifact = await verifyPiProductArtifact(options); + const common = { + samples: options.samples, + timeoutMs: options.timeoutMs, + stage: false, + keepStage: false, + reportPath: undefined, + providerFixturePath: undefined, + imagePath: undefined, + electronExecutablePath: artifact.artifact.executable, + cliPath: artifact.artifact.cliPath, + artifactLabel: 'final-makelore-product-artifact', + }; + const runtime = await runProbe(common, options.projectRoot); + if (runtime.result === 'fail') throw new Error('Final product Pi runtime lifecycle smoke failed'); + const providerContracts = await runLocalProviderContracts(common, options.projectRoot); + const extensionAndSubagent = await runExtensionSmoke(options.projectRoot, artifact); + const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + commit: process.env.MAKELORE_BUILD_COMMIT ?? null, + platform: { platform: platform(), arch: arch() }, + scope: { + finalProductArtifact: true, + actualPackagedPiProcess: true, + providerEndpoint: 'controlled-127.0.0.1-provider-shaped', + realProvider: false, + realTurnVerified: false, + realProviderDecision: 'explicitly-waived-accepted-risk', + }, + artifact, + runtime, + providerContracts, + extensionAndSubagent, + coverage: { + session: runtime.session.sessionIdPreserved, + prompt: runtime.session.promptAccepted, + tool: runtime.session.shellOutputMatched, + abort: runtime.localIsolation.abortIsolated, + settled: runtime.session.agentSettled, + reopen: runtime.session.sessionIdPreserved, + twoWorkerOverlap: providerContracts.protocols.every( + ({ qualification }) => qualification.overlapMs > 0, + ), + twoWorkerIsolation: providerContracts.protocols.every( + ({ qualification }) => qualification.distinctSessionIds, + ), + providerAbortIsolation: providerContracts.protocols.every( + ({ qualification }) => qualification.abortIsolation.passed, + ), + subagent: extensionAndSubagent.result === 'pass', + shutdown: runtime.performance.exitModes.every((mode) => mode === 'stdin-close'), + }, + result: 'pass', + crossPlatformReleaseReady: false, + releaseBlockers: [ + 'macOS x64 and macOS arm64 PI-150 final-artifact/runtime/resource/performance validation was skipped by explicit user direction', + ], + }; + if (Object.values(report.coverage).some((value) => value !== true)) { + throw new Error(`Final product Pi smoke coverage failed: ${JSON.stringify(report.coverage)}`); + } + if (options.reportPath) { + await mkdir(dirname(options.reportPath), { recursive: true }); + await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`); + } + return report; +} + +async function main() { + const options = parsePiArtifactVerifierArgs(process.argv.slice(2)); + if (options.help) { + process.stdout.write('Usage: node scripts/smoke-pi-real.mjs [--app-exe path] [--samples count] [--timeout-ms ms] [--report path]\n'); + return; + } + const report = await runRealPiSmoke(options); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} + +const isMain = process.argv[1] + && pathToFileURL(resolve(process.argv[1])).href === import.meta.url; +if (isMain) { + main().catch((error) => { + process.stderr.write(`${error.stack ?? error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/verify-pi-product-artifact.mjs b/scripts/verify-pi-product-artifact.mjs new file mode 100644 index 0000000..487925a --- /dev/null +++ b/scripts/verify-pi-product-artifact.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node + +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + defaultProductExecutable, + verifyPiProductArtifact, +} from './lib/pi-product-artifact.mjs'; +import { runProbe } from './probe-pi-runtime.mjs'; + +export function parsePiArtifactVerifierArgs(argv, projectRoot = process.cwd()) { + const options = { + projectRoot: resolve(projectRoot), + executable: undefined, + reportPath: undefined, + samples: 1, + timeoutMs: 10_000, + }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + const next = () => { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${argument} requires a value`); + index += 1; + return value; + }; + if (argument === '--app-exe') options.executable = resolve(next()); + else if (argument === '--report') options.reportPath = resolve(next()); + else if (argument === '--samples') options.samples = Number.parseInt(next(), 10); + else if (argument === '--timeout-ms') options.timeoutMs = Number.parseInt(next(), 10); + else if (argument === '--help') options.help = true; + else throw new Error(`Unknown argument: ${argument}`); + } + for (const [name, value] of [['--samples', options.samples], ['--timeout-ms', options.timeoutMs]]) { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); + } + options.executable ??= defaultProductExecutable(options.projectRoot); + return options; +} + +function printHelp() { + process.stdout.write('Usage: node scripts/verify-pi-product-artifact.mjs [options]\n\n'); + process.stdout.write(' --app-exe Final unpacked Makelore executable\n'); + process.stdout.write(' --samples Packaged get_state/runtime samples (default: 1)\n'); + process.stdout.write(' --timeout-ms Runtime probe timeout (default: 10000)\n'); + process.stdout.write(' --report Write structured JSON evidence\n'); +} + +export async function runPiArtifactVerifier(options) { + const artifact = await verifyPiProductArtifact(options); + const runtime = await runProbe({ + samples: options.samples, + timeoutMs: options.timeoutMs, + stage: false, + keepStage: false, + reportPath: undefined, + providerFixturePath: undefined, + imagePath: undefined, + electronExecutablePath: artifact.artifact.executable, + cliPath: artifact.artifact.cliPath, + artifactLabel: 'final-makelore-product-artifact', + }, options.projectRoot); + if (runtime.result === 'fail') throw new Error('Final product Pi get_state/runtime probe failed'); + const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + artifact, + runtime, + result: 'pass', + }; + if (options.reportPath) { + await mkdir(dirname(options.reportPath), { recursive: true }); + await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`); + } + return report; +} + +async function main() { + const options = parsePiArtifactVerifierArgs(process.argv.slice(2)); + if (options.help) { + printHelp(); + return; + } + const report = await runPiArtifactVerifier(options); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} + +const isMain = process.argv[1] + && pathToFileURL(resolve(process.argv[1])).href === import.meta.url; +if (isMain) { + main().catch((error) => { + process.stderr.write(`${error.stack ?? error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/tests/e2e/pi-coding-first-chat.spec.ts b/tests/e2e/pi-coding-first-chat.spec.ts index bbb5321..f292050 100644 --- a/tests/e2e/pi-coding-first-chat.spec.ts +++ b/tests/e2e/pi-coding-first-chat.spec.ts @@ -1,4 +1,5 @@ import type { ElectronApplication, Page } from 'playwright-core'; +import { appendFile } from 'node:fs/promises'; import { expect, getStableWindow, test } from './fixtures/electron'; type CapturedRequest = { @@ -116,7 +117,7 @@ async function installCodingFirstChatHost( initialized: true, agents: [agent], knowledgeDirectory: 'knowledge', - legacyConversationNotice: 'none', + legacyConversationNotice: featureComplete ? 'pending' : 'none', createdAt: now, updatedAt: now, }; @@ -150,20 +151,42 @@ async function installCodingFirstChatHost( title: conversation.title, model: { model: configuredModel, modelResolution: 'resolved' }, }, - nodes: featureComplete ? [{ - kind: 'subagent', - id: 'subagent-e2e', - runId: 'run-e2e-feature', - details: { - schema: 'subagent.v1', - dispatchId: 'dispatch-e2e', - mode: 'parallel', - tasks: [ - { taskId: 'task-reader', agentId: 'reader', toolProfile: 'read-only', status: 'complete', summary: 'Read complete' }, - { taskId: 'task-builder', agentId: 'builder', toolProfile: 'coding', status: 'running' }, - ], + nodes: featureComplete ? [ + { + kind: 'subagent', + id: 'subagent-e2e', + runId: 'run-e2e-feature', + details: { + schema: 'subagent.v1', + dispatchId: 'dispatch-e2e', + mode: 'parallel', + tasks: [ + { taskId: 'task-reader', agentId: 'reader', toolProfile: 'read-only', status: 'complete', summary: 'Read complete' }, + { taskId: 'task-builder', agentId: 'builder', toolProfile: 'coding', status: 'running' }, + ], + }, }, - }] : [], + { + kind: 'compaction', + id: 'compaction-e2e', + runId: 'run-e2e-feature', + source: 'automatic', + status: 'error', + willRetry: true, + summary: '保留了本轮关键上下文。', + }, + { + kind: 'tool', + id: 'browser-tool-e2e', + toolCallId: 'browser-call-e2e', + toolName: 'agent_browser', + title: '浏览器状态', + inputText: 'status', + status: 'complete', + output: [{ kind: 'text', id: 'browser-output-e2e', text: 'Browser ready', status: 'complete' }], + details: { schema: 'agent-browser.v1', action: 'status' }, + }, + ] : [], run: featureComplete ? { status: 'running', runId: 'run-e2e-feature', mode: 'prompt' } : { status: 'idle' }, queue: { items: featureComplete ? [{ id: 'queue-e2e', clientRequestId: 'request-queued', mode: 'follow-up', text: 'Queued follow-up', attachmentIds: [] }] : [] }, context: featureComplete @@ -196,6 +219,11 @@ async function installCodingFirstChatHost( run: { status: 'idle' }, queue: { items: [] }, pendingInteractions: [], + worker: { + status: 'error', + generation: 1, + error: { code: 'CODING_RUNTIME_START_FAILED', message: 'Worker stopped', recoverable: true }, + }, }; const respond = (json: unknown, status = 200) => ({ ok: true, @@ -294,6 +322,16 @@ async function installCodingFirstChatHost( if (path === `/api/coding/projects/config?projectId=${project.id}`) { return respond({ snapshot: { project, config, knowledgeFiles: [] } }); } + if (path === '/api/coding/projects/legacy-conversation-notice/acknowledge' + && method === 'POST') { + return respond({ + snapshot: { + project, + config: { ...config, legacyConversationNotice: 'acknowledged' }, + knowledgeFiles: [], + }, + }); + } if (path === `/api/coding/projects/conversations?projectId=${project.id}`) { return respond({ conversations: featureComplete @@ -416,15 +454,20 @@ test('first PI Conversation is editable under 500 ms and submits before runtime performance.now() - performance.getEntriesByName('pi-first-chat-start').at(-1)!.startTime )); expect(editableMs).toBeLessThan(500); + const performanceFragment = process.env.MAKELORE_PI_PERF_COMPOSER_FRAGMENT; + if (performanceFragment) { + await appendFile(performanceFragment, `${JSON.stringify({ editableMs })}\n`); + } const pixelPng = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', 'base64', ); + const largePng = Buffer.concat([pixelPng, Buffer.alloc(1024 * 1024)]); await page.getByTestId('coding-file-attachment-input').setInputFiles({ - name: 'pixel.png', + name: 'large.png', mimeType: 'image/png', - buffer: pixelPng, + buffer: largePng, }); await composer.fill('Build the first PI scene'); await expect(page.getByRole('button', { name: '发送' })).toBeEnabled(); @@ -453,7 +496,7 @@ test('first PI Conversation is editable under 500 ms and submits before runtime )); expect(uploads).toHaveLength(1); expect(uploads[0]).toMatchObject({ - byteLength: pixelPng.byteLength, + byteLength: largePng.byteLength, contentType: 'image/png', }); expect(state.captured.some((request) => ( @@ -466,7 +509,7 @@ test('first PI Conversation is editable under 500 ms and submits before runtime expect(prompt?.body?.attachments).toEqual([ { attachmentId: expect.any(String) }, ]); - expect(JSON.stringify(state.captured)).not.toContain(pixelPng.toString('base64')); + expect(JSON.stringify(prompt?.body)).not.toContain('data:image'); } finally { await releaseSnapshot(electronApp); } @@ -494,7 +537,13 @@ test('PI feature UI isolates Conversations and exposes queue, interaction, model await expect(page.getByTestId('coding-conversation-header')).toContainText('生成中'); await expect(page.getByTestId('coding-conversation-header')).toContainText('队列 1'); await expect(page.getByText('并行子任务')).toBeVisible(); + await expect(page.getByText('系统会自动重试。')).toBeVisible(); + await page.locator('[data-node-id="browser-tool-e2e"] > summary').click(); + await expect(page.getByText('浏览器操作 · status')).toBeVisible(); await expect(page.getByText('允许继续?')).toBeVisible(); + await expect(page.getByTestId('legacy-conversation-notice')).toBeVisible(); + await page.getByTestId('legacy-conversation-notice').getByRole('button', { name: '知道了' }).click(); + await expect(page.getByTestId('legacy-conversation-notice')).toHaveCount(0); await page.getByRole('button', { name: '确认', exact: true }).click(); const mode = page.getByRole('combobox', { name: '消息发送方式' }); @@ -517,6 +566,12 @@ test('PI feature UI isolates Conversations and exposes queue, interaction, model JSON.stringify(['account-e2e', 'model-b']), ); await expect(page.getByRole('textbox')).toHaveValue(''); + await page.getByRole('button', { name: '恢复' }).click(); + await page.getByRole('button', { name: '整理上下文' }).click(); + + await page.getByRole('button', { name: '打开编程工具' }).click(); + await page.getByRole('tab', { name: '技能' }).click(); + await expect(page.getByRole('dialog')).toContainText('Research'); await expect(page.getByText(/分享|取消分享|回滚|恢复回滚|待办|全局运行时/)).toHaveCount(0); const state = await readState(electronApp); @@ -525,6 +580,9 @@ test('PI feature UI isolates Conversations and exposes queue, interaction, model expect(state.captured.some((request) => request.path.includes('/interactions/') && request.path.endsWith('/respond'))).toBe(true); expect(state.captured.some((request) => request.path.endsWith('/commands'))).toBe(true); expect(state.captured.some((request) => request.path.endsWith('/changes'))).toBe(true); + expect(state.captured.some((request) => request.path.endsWith('/recover') && request.method === 'POST')).toBe(true); + expect(state.captured.some((request) => request.path.endsWith('/compact') && request.method === 'POST')).toBe(true); + expect(state.captured.some((request) => request.path.includes('legacy-conversation-notice'))).toBe(true); expect(state.captured.every((request) => !request.path.includes('/api/opencode/share'))).toBe(true); } finally { await releaseSnapshot(electronApp); diff --git a/tests/unit/coding-chat-pressure.test.tsx b/tests/unit/coding-chat-pressure.test.tsx index 348b5b3..d876770 100644 --- a/tests/unit/coding-chat-pressure.test.tsx +++ b/tests/unit/coding-chat-pressure.test.tsx @@ -1,5 +1,5 @@ import { act, render } from '@testing-library/react'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -36,6 +36,16 @@ function percentile95(values: number[]): number { return ordered[Math.ceil(ordered.length * 0.95) - 1] ?? Number.POSITIVE_INFINITY; } +function summarize(values: number[]): { samples: number; p50: number; p95: number; max: number } { + const ordered = [...values].sort((left, right) => left - right); + return { + samples: ordered.length, + p50: ordered[Math.ceil(ordered.length * 0.5) - 1] ?? Number.POSITIVE_INFINITY, + p95: percentile95(ordered), + max: ordered.at(-1) ?? Number.POSITIVE_INFINITY, + }; +} + function pressurePatch(seq: number): ConversationPatch { if (seq === 1) { return { @@ -203,7 +213,7 @@ describe('REN-008 coding timeline pressure', () => { latencies.push(performance.now() - startedAt); } const measuredReactCommits = reactCommits - initialCommits; - const p95Ms = percentile95(latencies); + const mainToReactMs = summarize(latencies); const metrics = { runtimePatchItems, patchBatches, @@ -211,7 +221,8 @@ describe('REN-008 coding timeline pressure', () => { rendererTransactions, reactCommits: measuredReactCommits, wireBytes, - mainToReactP95Ms: p95Ms, + mainToReactMs, + mainToReactP95Ms: mainToReactMs.p95, }; console.info('REN-008 metrics', metrics); expect(metrics.runtimePatchItems).toBe(100); @@ -225,6 +236,8 @@ describe('REN-008 coding timeline pressure', () => { .entriesByConversationId[conversation.id]?.reducer.snapshot?.cursor.seq).toBe(100); view.unmount(); unsubscribe(); + const reportPath = process.env.MAKELORE_PI_PERF_PRESSURE_FRAGMENT; + if (reportPath) await writeFile(reportPath, `${JSON.stringify(metrics, null, 2)}\n`); } finally { controller.abort(); await reader.cancel().catch(() => undefined); diff --git a/tests/unit/pi-product-artifact.test.ts b/tests/unit/pi-product-artifact.test.ts new file mode 100644 index 0000000..781871f --- /dev/null +++ b/tests/unit/pi-product-artifact.test.ts @@ -0,0 +1,100 @@ +// @vitest-environment node + +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { + assertNodeEngineCompatible, + collectAbsoluteManifestValues, + collectForbiddenResourcePaths, + defaultProductExecutable, + validatePiArtifactMetadata, +} from '../../scripts/lib/pi-product-artifact.mjs'; +import { parsePiArtifactVerifierArgs } from '../../scripts/verify-pi-product-artifact.mjs'; + +const roots: string[] = []; +const PI_PACKAGE = '@earendil-works/pi-coding-agent'; + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +function matchingMetadata() { + return { + rootPackage: { dependencies: { [PI_PACKAGE]: '0.84.2' } }, + lockfile: { + importers: { + '.': { dependencies: { [PI_PACKAGE]: { specifier: '0.84.2', version: '0.84.2(ws@8.20.0)' } } }, + }, + }, + packagedPackage: { dependencies: { [PI_PACKAGE]: '0.84.2' } }, + runtimePackage: { + name: PI_PACKAGE, + version: '0.84.2', + bin: { pi: 'dist/cli.js' }, + }, + manifest: { + runtime: { + packageName: PI_PACKAGE, + version: '0.84.2', + cliEntry: 'dist/cli.js', + nodeEngine: '>=22.19.0', + }, + target: { platform: 'win32', arch: 'x64' }, + }, + runtimePlatform: { platform: 'win32', arch: 'x64', node: '24.18.1' }, + }; +} + +describe('final Pi product artifact verification', () => { + it('requires the same pinned Pi version across package, lock, app, runtime, and manifest', () => { + expect(validatePiArtifactMetadata(matchingMetadata())).toMatchObject({ + expected: '0.84.2', + rootDependency: '0.84.2', + lockedVersion: '0.84.2', + manifest: '0.84.2', + }); + const mismatch = matchingMetadata(); + mismatch.packagedPackage.dependencies[PI_PACKAGE] = '0.84.1'; + expect(() => validatePiArtifactMetadata(mismatch)).toThrow('versions do not match'); + }); + + it('checks the packaged Node version against the exact supported engine shape', () => { + expect(() => assertNodeEngineCompatible('>=22.19.0', '24.18.1')).not.toThrow(); + expect(() => assertNodeEngineCompatible('>=22.19.0', '22.18.9')).toThrow('does not satisfy'); + expect(() => assertNodeEngineCompatible('^22.19.0', '24.18.1')).toThrow('Unsupported'); + }); + + it('rejects absolute manifest values and OpenCode-named artifact resources', async () => { + expect(collectAbsoluteManifestValues({ + entry: 'dist/cli.js', + asset: 'node_modules/example/file.wasm', + leaked: 'D:\\work\\pi-runtime', + })).toEqual([{ at: '$.leaked', value: 'D:\\work\\pi-runtime' }]); + + const root = await mkdtemp(path.join(tmpdir(), 'makelore-pi-artifact-test-')); + roots.push(root); + await mkdir(path.join(root, 'pi-runtime', 'node_modules', 'opencode-ai'), { recursive: true }); + await writeFile(path.join(root, 'pi-runtime', 'node_modules', 'opencode-ai', 'package.json'), '{}'); + expect(await collectForbiddenResourcePaths(root)).toEqual([ + 'pi-runtime/node_modules/opencode-ai', + ]); + }); + + it('uses final unpacked-product paths and strictly parses verifier options', () => { + expect(defaultProductExecutable('D:\\repo', 'win32')) + .toBe(path.resolve('D:\\repo', 'release', 'win-unpacked', 'Makelore.exe')); + expect(parsePiArtifactVerifierArgs([ + '--app-exe', 'release/custom/Makelore.exe', + '--samples', '5', + '--timeout-ms', '12000', + '--report', 'release/evidence/pi.json', + ], 'D:\\repo')).toMatchObject({ samples: 5, timeoutMs: 12_000 }); + expect(() => parsePiArtifactVerifierArgs(['--samples', '0'], 'D:\\repo')) + .toThrow('--samples must be a positive integer'); + expect(() => parsePiArtifactVerifierArgs(['--unknown'], 'D:\\repo')) + .toThrow('Unknown argument'); + }); +}); diff --git a/tests/unit/pi-release-docs.test.ts b/tests/unit/pi-release-docs.test.ts new file mode 100644 index 0000000..77385fa --- /dev/null +++ b/tests/unit/pi-release-docs.test.ts @@ -0,0 +1,31 @@ +import { readFile } from 'node:fs/promises'; +import { describe, expect, it } from 'vitest'; + +const root = process.cwd(); + +describe('Pi release documentation', () => { + it('keeps final-product commands and the external Provider waiver distinct', async () => { + const [readme, runbook] = await Promise.all([ + readFile(`${root}/README.md`, 'utf8'), + readFile(`${root}/docs/pi-runtime-release-runbook.md`, 'utf8'), + ]); + + expect(readme).toContain('pnpm run verify:artifact:pi'); + expect(readme).toContain('pnpm run smoke:pi:real'); + expect(readme).toContain('pnpm run perf:pi:release'); + expect(runbook).toContain('realTurnVerified=false'); + expect(runbook).toContain('Explicitly Waived / Accepted Risk'); + expect(runbook).toContain('不得写成 Pass'); + }); + + it('requires all release targets and documents the incompatible rollback boundary', async () => { + const runbook = await readFile(`${root}/docs/pi-runtime-release-runbook.md`, 'utf8'); + + for (const target of ['Windows x64', 'Linux x64', 'macOS x64', 'macOS arm64']) { + expect(runbook).toContain(target); + } + expect(runbook).toContain('缺少任一目标的独立证据时,结论只能是 `Blocked`'); + expect(runbook).toContain('旧版本不能读取或续写它们'); + expect(runbook).toContain('只支持完整应用版本回滚'); + }); +}); diff --git a/tests/unit/pi-release-performance-fragments.test.ts b/tests/unit/pi-release-performance-fragments.test.ts new file mode 100644 index 0000000..39983f2 --- /dev/null +++ b/tests/unit/pi-release-performance-fragments.test.ts @@ -0,0 +1,84 @@ +// @vitest-environment node + +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { createCodingConversationStore } from '../../electron/coding-projects/conversation-store'; +import { createCodingProjectAgent } from '../../electron/coding-projects/project-config'; +import { + createCodingProjectStore, + createLocalCodingProject, + createMemoryCodingProjectStorage, +} from '../../electron/coding-projects/project-store'; +import { summarizeMeasurements } from '../../scripts/probe-pi-runtime.mjs'; + +const roots: string[] = []; +const samples = Number.parseInt(process.env.MAKELORE_PI_PERF_SAMPLES ?? '10', 10); + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +describe('PI-150 local metadata performance fragments', () => { + it('reports project, Agent, and Conversation metadata distributions', async () => { + expect(samples).toBeGreaterThan(0); + const projectMs: number[] = []; + const agentMs: number[] = []; + const conversationMs: number[] = []; + for (let index = 0; index < samples; index += 1) { + const projectPath = await mkdtemp(path.join(tmpdir(), 'makelore-pi-release-perf-')); + roots.push(projectPath); + const store = createCodingProjectStore(createMemoryCodingProjectStorage(), { + createId: () => `project-${index}`, + now: () => '2026-08-24T00:00:00.000Z', + }); + + let startedAt = performance.now(); + await createLocalCodingProject({ + projectPath, + now: '2026-08-24T00:00:00.000Z', + }, store); + projectMs.push(performance.now() - startedAt); + + startedAt = performance.now(); + const agent = await createCodingProjectAgent(projectPath, { + id: `builder-${index}`, + avatarId: 'avatar-01', + roleName: '实现者', + name: `Builder ${index}`, + model: { accountId: 'account-a', modelId: 'model-a', thinkingLevel: 'medium' }, + modelResolution: 'resolved', + responsibility: { + mission: 'Implement', owns: [], boundaries: [], collaborators: [], principles: [], + }, + }, { now: '2026-08-24T00:00:00.000Z' }); + agentMs.push(performance.now() - startedAt); + + const conversations = createCodingConversationStore(projectPath, { + createId: () => `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`, + now: () => '2026-08-24T00:00:00.000Z', + }); + startedAt = performance.now(); + await conversations.create({ + agentId: agent.id, + title: `Conversation ${index}`, + model: agent.model, + modelResolution: agent.modelResolution, + }); + conversationMs.push(performance.now() - startedAt); + } + + const report = { + projectMetadataMs: summarizeMeasurements(projectMs), + agentMetadataMs: summarizeMeasurements(agentMs), + conversationMetadataMs: summarizeMeasurements(conversationMs), + }; + expect(report.projectMetadataMs.p95).toBeLessThanOrEqual(1_000); + expect(report.agentMetadataMs.p95).toBeLessThanOrEqual(500); + expect(report.conversationMetadataMs.p95).toBeLessThanOrEqual(500); + const reportPath = process.env.MAKELORE_PI_PERF_METADATA_FRAGMENT; + if (reportPath) await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); + }); +}); diff --git a/tests/unit/pi-runtime-bundle.test.ts b/tests/unit/pi-runtime-bundle.test.ts index 16dc7b1..04af983 100644 --- a/tests/unit/pi-runtime-bundle.test.ts +++ b/tests/unit/pi-runtime-bundle.test.ts @@ -103,6 +103,12 @@ describe('Pi runtime production bundler', () => { expect(packageJson.scripts.package).toContain('bundle-pi-runtime.mjs --release-targets'); expect(packageJson.scripts['package:stage:win-x64']) .toContain('bundle-pi-runtime.mjs --target win32-x64'); + expect(packageJson.scripts['verify:artifact:pi']) + .toBe('node scripts/verify-pi-product-artifact.mjs'); + expect(packageJson.scripts['smoke:pi:real']) + .toBe('node scripts/smoke-pi-real.mjs'); + expect(packageJson.scripts['perf:pi:release']) + .toBe('node scripts/run-pi-release-performance.mjs'); expect(builder.mac.extraResources).toContainEqual({ from: 'build/pi-runtime/darwin-${arch}', to: 'pi-runtime', diff --git a/tests/unit/pi-worker-process-real.test.ts b/tests/unit/pi-worker-process-real.test.ts index 0867be9..74bd43d 100644 --- a/tests/unit/pi-worker-process-real.test.ts +++ b/tests/unit/pi-worker-process-real.test.ts @@ -16,6 +16,11 @@ import { const scratchRoots: string[] = []; const packagedRuntimeRoot = process.env.MAKELORE_PI_STAGED_RUNTIME_ROOT; +function electronExecutableFromEnvironment(): string { + const requireFromProject = createRequire(resolve('package.json')); + return process.env.MAKELORE_PI_ELECTRON_EXECUTABLE ?? requireFromProject('electron') as string; +} + async function materializeActiveToolsProbe(root: string): Promise<{ extensionPath: string; resultPath: string; @@ -56,8 +61,7 @@ afterEach(async () => { describe('locked Pi worker process smoke', () => { it('loads the real Pi 0.84.2 entry through Electron Node and exits by closing stdin', async () => { - const requireFromProject = createRequire(resolve('package.json')); - const electronExecutable = requireFromProject('electron') as string; + const electronExecutable = electronExecutableFromEnvironment(); const packageRoot = realpathSync(resolve( 'node_modules', '@earendil-works', @@ -124,8 +128,7 @@ describe('locked Pi worker process smoke', () => { }, 15_000); it('starts a real ephemeral read-only child with the child extension role', async () => { - const requireFromProject = createRequire(resolve('package.json')); - const electronExecutable = requireFromProject('electron') as string; + const electronExecutable = electronExecutableFromEnvironment(); const packageRoot = realpathSync(resolve( 'node_modules', '@earendil-works', @@ -181,8 +184,7 @@ describe('locked Pi worker process smoke', () => { it.skipIf(!packagedRuntimeRoot)( 'starts the staged production-closure runtime as an ephemeral read-only child', async () => { - const requireFromProject = createRequire(resolve('package.json')); - const electronExecutable = requireFromProject('electron') as string; + const electronExecutable = electronExecutableFromEnvironment(); const packageRoot = realpathSync(packagedRuntimeRoot as string); const manifest = JSON.parse(await readFile(join(packageRoot, PI_RUNTIME_MANIFEST), 'utf8')) as { runtime?: { version?: string; cliEntry?: string }; @@ -246,8 +248,7 @@ describe('locked Pi worker process smoke', () => { it.skipIf(!packagedRuntimeRoot)( 'loads product tools in the staged production-closure parent runtime', async () => { - const requireFromProject = createRequire(resolve('package.json')); - const electronExecutable = requireFromProject('electron') as string; + const electronExecutable = electronExecutableFromEnvironment(); const packageRoot = realpathSync(packagedRuntimeRoot as string); const root = await mkdtemp(join(tmpdir(), 'makelore-pi-staged-product-tools-')); scratchRoots.push(root);