Files
makelore/electron/services/project-release-builder.ts

497 lines
21 KiB
TypeScript

import { createHash } from 'node:crypto';
import { createRequire } from 'node:module';
import { createReadStream, createWriteStream } from 'node:fs';
import { lstat, mkdir, mkdtemp, open, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { pipeline } from 'node:stream/promises';
import { dirname, join, relative, resolve, sep } from 'node:path';
import { tmpdir } from 'node:os';
import { app } from 'electron';
import { createStaticProjectPackage, type StaticProjectPackageSummary } from './project-packager';
import {
PublishRuntimeError,
resolvePublishRuntime,
runElectronNode,
type PublishRuntimeContext,
} from './publish-runtime';
import { createStaticArtifactSnapshot, type StaticArtifactSnapshot } from './static-release-server';
import { writeStoredZip } from './streaming-zip';
const require = createRequire(import.meta.url);
const AdmZip = require('adm-zip') as typeof import('adm-zip');
const FIXED_ZIP_TIME = new Date(1980, 0, 1);
const FILE_MODE = 0o100644;
const MAX_OUTPUT_FILES = 2_000;
const MAX_OUTPUT_BYTES = 50 * 1024 * 1024;
const activeReleaseProjects = new Set<string>();
export type ArtifactContract = {
schema_version: 1;
entry_path: 'index.html';
source_digest: string;
built_archive_digest: string;
artifact_digest: string;
file_count: number;
total_bytes: number;
files: Array<{ path: string; size: number; sha256: string }>;
security_profile: 'works-square-static-sandbox-v1';
toolchain: { client: 'makelore'; client_version: string; node: string; npm: string; vite: string };
};
export type PreparedRelease = {
sourceArchive: { path: string; name: 'project.zip'; bytes: Buffer; summary: StaticProjectPackageSummary };
builtArchive: { path: string; name: 'built-project.zip'; bytes: Buffer };
contract: ArtifactContract;
distRoot: string;
staticArtifact: StaticArtifactSnapshot;
dispose(): Promise<void>;
};
export type PreparedReleaseSummary = {
sourceArchive: { path: string; name: 'project.zip'; summary: StaticProjectPackageSummary };
builtArchive: { path: string; name: 'built-project.zip' };
contract: ArtifactContract;
dispose(): Promise<void>;
};
export class ProjectReleaseBuildError extends Error {
constructor(readonly code: 'LOCAL_BUILD_RUNTIME_UNAVAILABLE' | 'LOCAL_BUILD_FAILED' | 'LOCAL_BUILD_TIMEOUT' | 'LOCAL_BUILD_CANCELLED' | 'LOCAL_BUILD_OUTPUT_MISSING' | 'LOCAL_BUILD_BUSY') {
super(code);
this.name = 'ProjectReleaseBuildError';
}
}
const sha256 = (bytes: Buffer | string) => createHash('sha256').update(bytes).digest('hex');
const archivePath = (value: string) => sep === '/' ? value : value.split(sep).join('/');
const comparePaths = (left: string, right: string) => Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8'));
function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new ProjectReleaseBuildError('LOCAL_BUILD_CANCELLED');
}
function safeArchivePath(name: string): string[] | null {
if (!name || name.includes('\\') || name.startsWith('/') || name.includes('\0')) return null;
const parts = name.split('/');
if (parts.some((part) => !part || part === '.' || part === '..' || part.includes(':') || /[. ]$/.test(part))) return null;
if (parts.some((part) => /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part))) return null;
return parts;
}
const ZIP_EOCD_SIGNATURE = 0x06054b50;
const ZIP_CENTRAL_SIGNATURE = 0x02014b50;
const ZIP_LOCAL_SIGNATURE = 0x04034b50;
const ZIP_STORE_METHOD = 0;
const ZIP_MAX_COMMENT_BYTES = 0xffff;
const MAX_SOURCE_ARCHIVE_BYTES = 200 * 1024 * 1024;
type StoredZipCentralEntry = {
name: string;
parts: string[];
compressedSize: number;
uncompressedSize: number;
localOffset: number;
};
async function readZipBytes(
handle: Awaited<ReturnType<typeof open>>,
position: number,
length: number,
signal?: AbortSignal,
): Promise<Buffer> {
throwIfAborted(signal);
const result = Buffer.alloc(length);
let offset = 0;
while (offset < length) {
throwIfAborted(signal);
const read = await handle.read(result, offset, length - offset, position + offset);
if (read.bytesRead <= 0) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
offset += read.bytesRead;
}
return result;
}
function findZipEndOfCentralDirectory(tail: Buffer): number {
for (let offset = tail.length - 22; offset >= 0; offset -= 1) {
if (tail.readUInt32LE(offset) === ZIP_EOCD_SIGNATURE) return offset;
}
return -1;
}
async function readStoredZipEntries(
archive: string,
signal?: AbortSignal,
): Promise<{ handle: Awaited<ReturnType<typeof open>>; size: number; entries: StoredZipCentralEntry[] }> {
const handle = await open(archive, 'r');
try {
const stats = await handle.stat();
if (!stats.isFile() || stats.size > MAX_SOURCE_ARCHIVE_BYTES) {
throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
}
const tailLength = Math.min(stats.size, 22 + ZIP_MAX_COMMENT_BYTES);
const tail = await readZipBytes(handle, stats.size - tailLength, tailLength, signal);
const endOffset = findZipEndOfCentralDirectory(tail);
if (endOffset < 0) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const entryCount = tail.readUInt16LE(endOffset + 10);
const centralSize = tail.readUInt32LE(endOffset + 12);
const centralOffset = tail.readUInt32LE(endOffset + 16);
if (
tail.readUInt16LE(endOffset + 4) !== 0
|| tail.readUInt16LE(endOffset + 6) !== 0
|| entryCount === 0
|| entryCount > MAX_OUTPUT_FILES
|| centralSize > stats.size
|| centralOffset > stats.size
|| centralOffset + centralSize > stats.size
) {
throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
}
const entries: StoredZipCentralEntry[] = [];
let cursor = centralOffset;
const centralEnd = centralOffset + centralSize;
for (let index = 0; index < entryCount; index += 1) {
throwIfAborted(signal);
if (cursor + 46 > centralEnd) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const header = await readZipBytes(handle, cursor, 46, signal);
if (header.readUInt32LE(0) !== ZIP_CENTRAL_SIGNATURE) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const nameLength = header.readUInt16LE(28);
const extraLength = header.readUInt16LE(30);
const commentLength = header.readUInt16LE(32);
const recordLength = 46 + nameLength + extraLength + commentLength;
if (cursor + recordLength > centralEnd) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const name = (await readZipBytes(handle, cursor + 46, nameLength, signal)).toString('utf8');
const parts = safeArchivePath(name);
const flags = header.readUInt16LE(8);
const method = header.readUInt16LE(10);
const compressedSize = header.readUInt32LE(20);
const uncompressedSize = header.readUInt32LE(24);
const localOffset = header.readUInt32LE(42);
const unixMode = (header.readUInt32LE(38) >>> 16) & 0xffff;
const unixType = unixMode & 0xf000;
if (
!parts
|| (flags & 0x0001) !== 0
|| method !== ZIP_STORE_METHOD
|| compressedSize === 0xffffffff
|| uncompressedSize === 0xffffffff
|| localOffset >= centralOffset
|| (unixType !== 0 && unixType !== 0x8000)
) {
throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
}
entries.push({ name, parts, compressedSize, uncompressedSize, localOffset });
cursor += recordLength;
}
if (cursor !== centralEnd) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
return { handle, size: stats.size, entries };
} catch (error) {
await handle.close().catch(() => undefined);
throw error;
}
}
async function extractSnapshot(archive: string, target: string, signal?: AbortSignal): Promise<void> {
const zip = await readStoredZipEntries(archive, signal);
const destinations = new Set<string>();
let totalBytes = 0;
try {
for (const entry of zip.entries) {
throwIfAborted(signal);
const destinationKey = entry.name.toLowerCase();
if (destinations.has(destinationKey)) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
destinations.add(destinationKey);
if (entry.uncompressedSize > MAX_SOURCE_ARCHIVE_BYTES || entry.uncompressedSize !== entry.compressedSize) {
throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
}
totalBytes += entry.uncompressedSize;
if (totalBytes > MAX_SOURCE_ARCHIVE_BYTES) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const localHeader = await readZipBytes(zip.handle, entry.localOffset, 30, signal);
if (localHeader.readUInt32LE(0) !== ZIP_LOCAL_SIGNATURE) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const localFlags = localHeader.readUInt16LE(6);
const localMethod = localHeader.readUInt16LE(8);
const localNameLength = localHeader.readUInt16LE(26);
const localExtraLength = localHeader.readUInt16LE(28);
if ((localFlags & 0x0001) !== 0 || localMethod !== ZIP_STORE_METHOD) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const localName = (await readZipBytes(zip.handle, entry.localOffset + 30, localNameLength, signal)).toString('utf8');
if (localName !== entry.name) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const dataStart = entry.localOffset + 30 + localNameLength + localExtraLength;
const dataEnd = dataStart + entry.compressedSize;
if (dataStart > zip.size || dataEnd > zip.size || dataEnd < dataStart) throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
const destination = join(target, ...entry.parts);
await mkdir(dirname(destination), { recursive: true });
try {
if (entry.compressedSize === 0) {
await writeFile(destination, Buffer.alloc(0), { flag: 'wx', mode: 0o600 });
} else {
const input = zip.handle.createReadStream({ start: dataStart, end: dataEnd - 1, autoClose: false });
const output = createWriteStream(destination, { flags: 'wx', mode: 0o600 });
await pipeline(input, output, { signal });
}
const written = await lstat(destination);
if (!written.isFile() || written.size !== entry.uncompressedSize) throw new Error('size mismatch');
} catch {
await rm(destination, { force: true }).catch(() => undefined);
throwIfAborted(signal);
throw new ProjectReleaseBuildError('LOCAL_BUILD_FAILED');
}
}
} finally {
await zip.handle.close().catch(() => undefined);
}
}
async function collectOutput(root: string, signal?: AbortSignal): Promise<Array<{ path: string; bytes: Buffer }>> {
const files: Array<{ path: string; bytes: Buffer }> = [];
let total = 0;
async function walk(directory: string): Promise<void> {
const entries = await readdir(directory, { withFileTypes: true });
entries.sort((a, b) => comparePaths(a.name, b.name));
for (const entry of entries) {
throwIfAborted(signal);
const absolute = join(directory, entry.name);
const stats = await lstat(absolute);
if (stats.isSymbolicLink() || (!stats.isFile() && !stats.isDirectory())) throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
if (stats.isDirectory()) await walk(absolute);
else {
total += stats.size;
if (files.length + 1 > MAX_OUTPUT_FILES || total > MAX_OUTPUT_BYTES) throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
files.push({ path: archivePath(relative(root, absolute)), bytes: await readFile(absolute) });
}
}
}
try { await walk(root); } catch (error) {
if (error instanceof ProjectReleaseBuildError) throw error;
throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
}
files.sort((a, b) => comparePaths(a.path, b.path));
if (files.some((file) => file.path.toLowerCase() === 'release.json')) throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
const index = files.find((file) => file.path === 'index.html');
if (!index) throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
try {
if (!new TextDecoder('utf-8', { fatal: true }).decode(index.bytes).trim()) throw new Error('empty');
} catch { throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING'); }
return files;
}
async function hashFile(filePath: string, signal?: AbortSignal): Promise<{ bytes: number; sha256: string }> {
const hash = createHash('sha256');
let bytes = 0;
const stream = createReadStream(filePath);
try {
for await (const chunk of stream) {
throwIfAborted(signal);
const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
bytes += value.length;
hash.update(value);
}
} finally {
stream.destroy();
}
return { bytes, sha256: hash.digest('hex') };
}
async function collectOutputMetadata(
root: string,
signal?: AbortSignal,
): Promise<Array<{ path: string; filePath: string; size: number; sha256: string }>> {
const files: Array<{ path: string; filePath: string; size: number; sha256: string }> = [];
let total = 0;
async function walk(directory: string): Promise<void> {
const entries = await readdir(directory, { withFileTypes: true });
entries.sort((a, b) => comparePaths(a.name, b.name));
for (const entry of entries) {
throwIfAborted(signal);
const absolute = join(directory, entry.name);
const stats = await lstat(absolute);
if (stats.isSymbolicLink() || (!stats.isFile() && !stats.isDirectory())) {
throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
}
if (stats.isDirectory()) {
await walk(absolute);
continue;
}
const path = archivePath(relative(root, absolute));
if (files.length + 1 > MAX_OUTPUT_FILES || total + stats.size > MAX_OUTPUT_BYTES) {
throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
}
const digest = await hashFile(absolute, signal);
if (digest.bytes !== stats.size) throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
total += digest.bytes;
files.push({ path, filePath: absolute, size: digest.bytes, sha256: digest.sha256 });
}
}
try {
await walk(root);
} catch (error) {
if (error instanceof ProjectReleaseBuildError) throw error;
throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
}
files.sort((a, b) => comparePaths(a.path, b.path));
if (files.some((file) => file.path.toLowerCase() === 'release.json')) {
throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
}
const index = files.find((file) => file.path === 'index.html');
if (!index || index.size === 0) throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING');
return files;
}
export type ReleaseBuildProgress = {
phase: 'packaging' | 'installing' | 'building' | 'archiving' | 'complete';
percent: number;
};
export type ReleaseBuildRuntimeContext = Partial<PublishRuntimeContext> & {
/** Persistent npm cache supplied by Main so utility processes never need app.getPath(). */
npmCacheDir?: string;
};
type ReleaseBuildInput = {
projectPath: string;
clientVersion: string;
signal?: AbortSignal;
onProgress?: (progress: ReleaseBuildProgress) => void;
runtimeContext?: ReleaseBuildRuntimeContext;
};
async function prepareProjectReleaseInternal(
input: ReleaseBuildInput,
retainBuffers: boolean,
): Promise<PreparedRelease | PreparedReleaseSummary> {
const projectKey = resolve(input.projectPath);
if (activeReleaseProjects.has(projectKey)) {
throw new ProjectReleaseBuildError('LOCAL_BUILD_BUSY');
}
activeReleaseProjects.add(projectKey);
const progress = (phase: ReleaseBuildProgress['phase'], percent: number): void => {
input.onProgress?.({ phase, percent });
throwIfAborted(input.signal);
};
let taskRoot: string;
try {
taskRoot = await mkdtemp(join(tmpdir(), 'makelore-release-'));
} catch (error) {
activeReleaseProjects.delete(projectKey);
throw error;
}
try {
const sourcePath = join(taskRoot, 'project.zip');
progress('packaging', 5);
const summary = await createStaticProjectPackage({ projectPath: input.projectPath, archivePath: sourcePath });
progress('packaging', 20);
const sourceBytes = retainBuffers ? await readFile(sourcePath) : undefined;
const snapshot = join(taskRoot, 'snapshot');
const distRoot = join(taskRoot, 'owned-dist');
const npmUserConfig = join(taskRoot, 'empty-npmrc');
const npmCache = input.runtimeContext?.npmCacheDir
?? join(app.getPath('userData'), 'npm-cache');
await mkdir(snapshot, { recursive: true });
await writeFile(npmUserConfig, '', 'utf8');
await extractSnapshot(sourcePath, snapshot, input.signal);
progress('installing', 25);
const runtime = await resolvePublishRuntime(input.runtimeContext);
await runElectronNode({
args: [runtime.npmCli, 'ci', '--ignore-scripts', '--no-audit', '--no-fund', '--prefer-offline', '--progress=false', '--userconfig', npmUserConfig, '--cache', npmCache],
cwd: snapshot,
deadlineMs: 5 * 60_000,
signal: input.signal,
});
progress('installing', 55);
let viteVersion = '';
try {
const vitePackage = JSON.parse(await readFile(join(snapshot, 'node_modules', 'vite', 'package.json'), 'utf8')) as { version?: unknown };
if (typeof vitePackage.version !== 'string') throw new Error('missing vite version');
viteVersion = vitePackage.version;
} catch { throw new ProjectReleaseBuildError('LOCAL_BUILD_OUTPUT_MISSING'); }
// Vite config and plugins execute with desktop-user authority. This build is not a sandbox or trust proof.
await runElectronNode({
args: [join(snapshot, 'node_modules', 'vite', 'bin', 'vite.js'), 'build', '--base', './', '--outDir', distRoot, '--emptyOutDir'],
cwd: snapshot,
deadlineMs: 3 * 60_000,
signal: input.signal,
});
progress('building', 80);
progress('archiving', 90);
const builtPath = join(taskRoot, 'built-project.zip');
let files: Array<{ path: string; size: number; sha256: string }>;
let builtBytes: Buffer | undefined;
let builtArchiveDigest: string;
let staticArtifact: StaticArtifactSnapshot | undefined;
if (retainBuffers) {
const output = await collectOutput(distRoot, input.signal);
files = output.map(({ path, bytes }) => ({ path, size: bytes.length, sha256: sha256(bytes) }));
const zip = new AdmZip();
for (const file of output) {
zip.addFile(file.path, file.bytes, '', FILE_MODE);
const entry = zip.getEntry(file.path);
if (entry) entry.header.time = FIXED_ZIP_TIME;
}
builtBytes = zip.toBuffer();
await writeFile(builtPath, builtBytes);
builtArchiveDigest = sha256(builtBytes);
staticArtifact = createStaticArtifactSnapshot(output);
} else {
const metadata = await collectOutputMetadata(distRoot, input.signal);
files = metadata.map(({ path, size, sha256: digest }) => ({ path, size, sha256: digest }));
const streamed = await writeStoredZip(
metadata.map((file) => ({ path: file.path, source: { filePath: file.filePath } })),
builtPath,
{ signal: input.signal, date: FIXED_ZIP_TIME },
);
builtArchiveDigest = streamed.sha256;
}
const totalBytes = files.reduce((sum, file) => sum + file.size, 0);
// Matches the server's canonical JSON: sorted files, sorted object keys, no whitespace.
const artifactDigest = sha256(JSON.stringify(files.map((file) => ({
path: file.path,
sha256: file.sha256,
size: file.size,
}))));
const contract: ArtifactContract = {
schema_version: 1,
entry_path: 'index.html',
source_digest: sourceBytes ? sha256(sourceBytes) : (await hashFile(sourcePath, input.signal)).sha256,
built_archive_digest: builtArchiveDigest,
artifact_digest: artifactDigest,
file_count: files.length,
total_bytes: totalBytes,
files,
security_profile: 'works-square-static-sandbox-v1',
toolchain: { client: 'makelore', client_version: input.clientVersion, node: runtime.nodeVersion, npm: runtime.npmVersion, vite: viteVersion },
};
const dispose = async (): Promise<void> => { await rm(taskRoot, { recursive: true, force: true }); };
const prepared: PreparedRelease | PreparedReleaseSummary = retainBuffers
? {
sourceArchive: { path: sourcePath, name: 'project.zip', bytes: sourceBytes!, summary },
builtArchive: { path: builtPath, name: 'built-project.zip', bytes: builtBytes! },
distRoot,
staticArtifact: staticArtifact!,
contract,
dispose,
}
: {
sourceArchive: { path: sourcePath, name: 'project.zip', summary },
builtArchive: { path: builtPath, name: 'built-project.zip' },
contract,
dispose,
};
progress('complete', 100);
activeReleaseProjects.delete(projectKey);
return prepared;
} catch (error) {
activeReleaseProjects.delete(projectKey);
await rm(taskRoot, { recursive: true, force: true }).catch(() => undefined);
if (error instanceof ProjectReleaseBuildError) throw error;
if (error instanceof PublishRuntimeError) throw new ProjectReleaseBuildError(error.code);
throw error;
}
}
export async function prepareProjectRelease(input: ReleaseBuildInput): Promise<PreparedRelease> {
return await prepareProjectReleaseInternal(input, true) as PreparedRelease;
}
export async function prepareProjectReleaseSummary(input: ReleaseBuildInput): Promise<PreparedReleaseSummary> {
return await prepareProjectReleaseInternal(input, false) as PreparedReleaseSummary;
}