perf: optimize app startup and background lifecycles
This commit is contained in:
inman
262
electron/services/streaming-zip.ts
Normal file
262
electron/services/streaming-zip.ts
Normal file
@@ -0,0 +1,262 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createWriteStream } from 'node:fs';
|
||||
import { open } from 'node:fs/promises';
|
||||
|
||||
const ZIP_LOCAL_FILE_HEADER = 0x04034b50;
|
||||
const ZIP_CENTRAL_DIRECTORY_HEADER = 0x02014b50;
|
||||
const ZIP_DATA_DESCRIPTOR = 0x08074b50;
|
||||
const ZIP_END_OF_CENTRAL_DIRECTORY = 0x06054b50;
|
||||
const ZIP_VERSION = 20;
|
||||
const ZIP_DATA_DESCRIPTOR_FLAG = 0x0008;
|
||||
const ZIP_STORE_METHOD = 0;
|
||||
const ZIP_FILE_MODE = 0o100644;
|
||||
const FIXED_ZIP_TIME = new Date(1980, 0, 1, 0, 0, 0, 0);
|
||||
|
||||
export type StreamingZipEntry = {
|
||||
path: string;
|
||||
source: { filePath: string } | { bytes: Buffer };
|
||||
beforeRead?: () => Promise<void>;
|
||||
afterRead?: (result: { crc32: number; size: number }) => Promise<void>;
|
||||
};
|
||||
|
||||
export type StreamingZipResult = {
|
||||
archiveBytes: number;
|
||||
sha256: string;
|
||||
};
|
||||
|
||||
type CentralEntry = {
|
||||
path: string;
|
||||
crc32: number;
|
||||
size: number;
|
||||
offset: number;
|
||||
};
|
||||
|
||||
type OutputState = {
|
||||
error: Error | null;
|
||||
};
|
||||
|
||||
const CRC_TABLE = (() => {
|
||||
const table = new Uint32Array(256);
|
||||
for (let index = 0; index < table.length; index += 1) {
|
||||
let value = index;
|
||||
for (let bit = 0; bit < 8; bit += 1) {
|
||||
value = (value & 1) === 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
||||
}
|
||||
table[index] = value >>> 0;
|
||||
}
|
||||
return table;
|
||||
})();
|
||||
|
||||
function updateCrc32(current: number, chunk: Uint8Array): number {
|
||||
let value = current ^ 0xffffffff;
|
||||
for (const byte of chunk) value = CRC_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8);
|
||||
return (value ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function dosDateTime(date: Date): { time: number; day: number } {
|
||||
const year = Math.max(1980, Math.min(2107, date.getFullYear()));
|
||||
return {
|
||||
time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2),
|
||||
day: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
|
||||
};
|
||||
}
|
||||
|
||||
function pathBytes(path: string): Buffer {
|
||||
const bytes = Buffer.from(path, 'utf8');
|
||||
if (bytes.length > 0xffff) throw new Error('ZIP entry path is too long');
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function localHeader(path: Buffer, date: Date): Buffer {
|
||||
const header = Buffer.alloc(30 + path.length);
|
||||
const dos = dosDateTime(date);
|
||||
header.writeUInt32LE(ZIP_LOCAL_FILE_HEADER, 0);
|
||||
header.writeUInt16LE(ZIP_VERSION, 4);
|
||||
header.writeUInt16LE(ZIP_DATA_DESCRIPTOR_FLAG, 6);
|
||||
header.writeUInt16LE(ZIP_STORE_METHOD, 8);
|
||||
header.writeUInt16LE(dos.time, 10);
|
||||
header.writeUInt16LE(dos.day, 12);
|
||||
header.writeUInt16LE(0, 14);
|
||||
header.writeUInt32LE(0, 18);
|
||||
header.writeUInt32LE(0, 22);
|
||||
header.writeUInt16LE(path.length, 26);
|
||||
header.writeUInt16LE(0, 28);
|
||||
path.copy(header, 30);
|
||||
return header;
|
||||
}
|
||||
|
||||
function dataDescriptor(crc32: number, size: number): Buffer {
|
||||
const descriptor = Buffer.alloc(16);
|
||||
descriptor.writeUInt32LE(ZIP_DATA_DESCRIPTOR, 0);
|
||||
descriptor.writeUInt32LE(crc32 >>> 0, 4);
|
||||
descriptor.writeUInt32LE(size >>> 0, 8);
|
||||
descriptor.writeUInt32LE(size >>> 0, 12);
|
||||
return descriptor;
|
||||
}
|
||||
|
||||
function centralHeader(entry: CentralEntry, date: Date): Buffer {
|
||||
const path = pathBytes(entry.path);
|
||||
const header = Buffer.alloc(46 + path.length);
|
||||
const dos = dosDateTime(date);
|
||||
header.writeUInt32LE(ZIP_CENTRAL_DIRECTORY_HEADER, 0);
|
||||
header.writeUInt16LE(ZIP_VERSION, 4);
|
||||
header.writeUInt16LE(ZIP_VERSION, 6);
|
||||
header.writeUInt16LE(ZIP_DATA_DESCRIPTOR_FLAG, 8);
|
||||
header.writeUInt16LE(ZIP_STORE_METHOD, 10);
|
||||
header.writeUInt16LE(dos.time, 12);
|
||||
header.writeUInt16LE(dos.day, 14);
|
||||
header.writeUInt32LE(entry.crc32 >>> 0, 16);
|
||||
header.writeUInt32LE(entry.size >>> 0, 20);
|
||||
header.writeUInt32LE(entry.size >>> 0, 24);
|
||||
header.writeUInt16LE(path.length, 28);
|
||||
header.writeUInt16LE(0, 30);
|
||||
header.writeUInt16LE(0, 32);
|
||||
header.writeUInt16LE(0, 34);
|
||||
header.writeUInt16LE(0, 36);
|
||||
header.writeUInt32LE((ZIP_FILE_MODE << 16) >>> 0, 38);
|
||||
header.writeUInt32LE(entry.offset >>> 0, 42);
|
||||
path.copy(header, 46);
|
||||
return header;
|
||||
}
|
||||
|
||||
function endOfCentralDirectory(count: number, size: number, offset: number): Buffer {
|
||||
const footer = Buffer.alloc(22);
|
||||
footer.writeUInt32LE(ZIP_END_OF_CENTRAL_DIRECTORY, 0);
|
||||
footer.writeUInt16LE(0, 4);
|
||||
footer.writeUInt16LE(0, 6);
|
||||
footer.writeUInt16LE(count, 8);
|
||||
footer.writeUInt16LE(count, 10);
|
||||
footer.writeUInt32LE(size, 12);
|
||||
footer.writeUInt32LE(offset, 16);
|
||||
footer.writeUInt16LE(0, 20);
|
||||
return footer;
|
||||
}
|
||||
|
||||
async function writeChunk(
|
||||
output: ReturnType<typeof createWriteStream>,
|
||||
hash: ReturnType<typeof createHash>,
|
||||
chunk: Buffer,
|
||||
state: OutputState,
|
||||
): Promise<void> {
|
||||
if (state.error) throw state.error;
|
||||
hash.update(chunk);
|
||||
if (output.write(chunk)) return;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const cleanup = () => {
|
||||
output.removeListener('drain', onDrain);
|
||||
output.removeListener('error', onError);
|
||||
};
|
||||
output.once('drain', onDrain);
|
||||
output.once('error', onError);
|
||||
});
|
||||
if (state.error) throw state.error;
|
||||
}
|
||||
|
||||
async function writeEntryData(
|
||||
output: ReturnType<typeof createWriteStream>,
|
||||
hash: ReturnType<typeof createHash>,
|
||||
entry: StreamingZipEntry,
|
||||
state: OutputState,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ crc32: number; size: number }> {
|
||||
let crc32 = 0;
|
||||
let size = 0;
|
||||
const writeData = async (chunk: Buffer): Promise<void> => {
|
||||
if (signal?.aborted) throw new Error('ZIP_WRITE_CANCELLED');
|
||||
crc32 = updateCrc32(crc32, chunk);
|
||||
size += chunk.length;
|
||||
if (size > 0xffffffff) throw new Error('ZIP entry exceeds 4 GiB limit');
|
||||
await writeChunk(output, hash, chunk, state);
|
||||
};
|
||||
|
||||
if ('bytes' in entry.source) {
|
||||
await entry.beforeRead?.();
|
||||
await writeData(entry.source.bytes);
|
||||
} else {
|
||||
// Open the file before reading so a later path replacement cannot redirect
|
||||
// the stream outside the package scan's validated inode. The explicit
|
||||
// open also lets callers validate a stable-file race immediately before
|
||||
// the descriptor is acquired.
|
||||
const handle = await open(entry.source.filePath, 'r');
|
||||
try {
|
||||
await entry.beforeRead?.();
|
||||
const input = handle.createReadStream({ autoClose: false });
|
||||
for await (const chunk of input) {
|
||||
await writeData(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
await entry.afterRead?.({ crc32, size });
|
||||
return { crc32, size };
|
||||
}
|
||||
|
||||
/** Writes a deterministic, uncompressed ZIP without retaining file contents. */
|
||||
export async function writeStoredZip(
|
||||
entries: readonly StreamingZipEntry[],
|
||||
archivePath: string,
|
||||
options: { signal?: AbortSignal; date?: Date } = {},
|
||||
): Promise<StreamingZipResult> {
|
||||
if (entries.length > 0xffff) throw new Error('ZIP contains too many entries');
|
||||
const date = options.date ?? FIXED_ZIP_TIME;
|
||||
const output = createWriteStream(archivePath, { mode: 0o600 });
|
||||
const outputState: OutputState = { error: null };
|
||||
output.on('error', (error) => {
|
||||
outputState.error = error;
|
||||
});
|
||||
const hash = createHash('sha256');
|
||||
const central: CentralEntry[] = [];
|
||||
let offset = 0;
|
||||
|
||||
try {
|
||||
for (const entry of entries) {
|
||||
const path = pathBytes(entry.path);
|
||||
const start = offset;
|
||||
const header = localHeader(path, date);
|
||||
await writeChunk(output, hash, header, outputState);
|
||||
offset += header.length;
|
||||
const data = await writeEntryData(output, hash, entry, outputState, options.signal);
|
||||
offset += data.size;
|
||||
const descriptor = dataDescriptor(data.crc32, data.size);
|
||||
await writeChunk(output, hash, descriptor, outputState);
|
||||
offset += descriptor.length;
|
||||
central.push({ path: entry.path, crc32: data.crc32, size: data.size, offset: start });
|
||||
}
|
||||
|
||||
const centralOffset = offset;
|
||||
for (const entry of central) {
|
||||
const header = centralHeader(entry, date);
|
||||
await writeChunk(output, hash, header, outputState);
|
||||
offset += header.length;
|
||||
}
|
||||
const footer = endOfCentralDirectory(central.length, offset - centralOffset, centralOffset);
|
||||
await writeChunk(output, hash, footer, outputState);
|
||||
offset += footer.length;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const cleanup = () => output.removeListener('error', onError);
|
||||
output.once('error', onError);
|
||||
output.end(() => {
|
||||
cleanup();
|
||||
if (outputState.error) reject(outputState.error);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
return { archiveBytes: offset, sha256: hash.digest('hex') };
|
||||
} catch (error) {
|
||||
output.destroy();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user