feat: ship deterministic parser and lifecycle release
This commit is contained in:
inman committed
1 parent
208c434b31
commit
7b5d855b09
176 files changed
+20248
-1362
No files matched your search
@@ -0,0 +1,309 @@
|
||||
import { lookup as dnsLookup } from 'node:dns/promises';
|
||||
import { request as httpsRequest } from 'node:https';
|
||||
import { isIP } from 'node:net';
|
||||
import { basename } from 'node:path';
|
||||
import { sha256Bytes } from './crypto.js';
|
||||
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/i;
|
||||
const BASE64_PATTERN = /^[A-Za-z0-9+/_-]*={0,2}$/;
|
||||
const ALLOWED_ROSTER_EXTENSIONS = new Set(['.xls', '.xlsx']);
|
||||
const MAX_REDIRECTS = 2;
|
||||
const DOWNLOAD_TIMEOUT_MS = 30_000;
|
||||
|
||||
export interface TaskInputAttachmentInput {
|
||||
fileName: string;
|
||||
contentType: string;
|
||||
content: Buffer;
|
||||
declaredSize?: number;
|
||||
declaredSha256?: string;
|
||||
source: 'manual' | 'agentbus';
|
||||
}
|
||||
|
||||
export interface EncodedTaskInputAttachment {
|
||||
name: string;
|
||||
content_type?: string;
|
||||
size?: number;
|
||||
sha256?: string;
|
||||
content_base64: string;
|
||||
}
|
||||
|
||||
export interface AgentBusInputAttachmentReference {
|
||||
name: string;
|
||||
contentType: string;
|
||||
size?: number;
|
||||
sha256?: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export class InputAttachmentError extends Error {
|
||||
constructor(
|
||||
public readonly code: string,
|
||||
message: string,
|
||||
public readonly details: Record<string, unknown> = {}
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'InputAttachmentError';
|
||||
}
|
||||
}
|
||||
|
||||
function extensionOf(fileName: string): string {
|
||||
const lower = fileName.toLowerCase();
|
||||
if (lower.endsWith('.xlsx')) return '.xlsx';
|
||||
if (lower.endsWith('.xls')) return '.xls';
|
||||
return '';
|
||||
}
|
||||
|
||||
export function normalizeInputAttachmentFileName(value: unknown): string {
|
||||
const normalized = basename(String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, '_'))
|
||||
.trim()
|
||||
.slice(0, 200);
|
||||
if (!normalized || !ALLOWED_ROSTER_EXTENSIONS.has(extensionOf(normalized))) {
|
||||
throw new InputAttachmentError('roster_file_type_unsupported', '名单附件必须是 .xls 或 .xlsx 文件。');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeContentType(value: unknown): string {
|
||||
return String(value ?? '')
|
||||
.replace(/[\r\n]/g, '')
|
||||
.trim()
|
||||
.slice(0, 200) || 'application/octet-stream';
|
||||
}
|
||||
|
||||
function normalizeDeclaredSize(value: unknown, maxBytes: number): number | undefined {
|
||||
if (value === undefined || value === null || value === '') return undefined;
|
||||
const size = Number(value);
|
||||
if (!Number.isInteger(size) || size < 1) {
|
||||
throw new InputAttachmentError('roster_file_size_invalid', '名单附件大小声明无效。');
|
||||
}
|
||||
if (size > maxBytes) {
|
||||
throw new InputAttachmentError('roster_file_too_large', '名单附件超过大小限制。', { max_bytes: maxBytes });
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
function normalizeDeclaredSha256(value: unknown): string | undefined {
|
||||
const normalized = String(value ?? '').trim().toLowerCase();
|
||||
if (!normalized) return undefined;
|
||||
if (!SHA256_PATTERN.test(normalized)) {
|
||||
throw new InputAttachmentError('roster_file_sha256_invalid', '名单附件 SHA-256 声明无效。');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function validateAttachmentBytes(
|
||||
content: Buffer,
|
||||
maxBytes: number,
|
||||
declaredSize?: number,
|
||||
declaredSha256?: string
|
||||
): void {
|
||||
if (!content.byteLength) throw new InputAttachmentError('roster_file_empty', '名单附件为空。');
|
||||
if (content.byteLength > maxBytes) {
|
||||
throw new InputAttachmentError('roster_file_too_large', '名单附件超过大小限制。', { max_bytes: maxBytes });
|
||||
}
|
||||
if (declaredSize !== undefined && content.byteLength !== declaredSize) {
|
||||
throw new InputAttachmentError('roster_file_size_mismatch', '名单附件实际大小与声明不一致。', {
|
||||
declared_size: declaredSize,
|
||||
actual_size: content.byteLength
|
||||
});
|
||||
}
|
||||
const digest = sha256Bytes(content);
|
||||
if (declaredSha256 && digest !== declaredSha256) {
|
||||
throw new InputAttachmentError('roster_file_sha256_mismatch', '名单附件 SHA-256 校验失败。');
|
||||
}
|
||||
}
|
||||
|
||||
export function decodeInlineInputAttachment(
|
||||
value: EncodedTaskInputAttachment,
|
||||
maxBytes: number,
|
||||
source: TaskInputAttachmentInput['source'] = 'manual'
|
||||
): TaskInputAttachmentInput {
|
||||
const fileName = normalizeInputAttachmentFileName(value.name);
|
||||
const declaredSize = normalizeDeclaredSize(value.size, maxBytes);
|
||||
const declaredSha256 = normalizeDeclaredSha256(value.sha256);
|
||||
const encoded = String(value.content_base64 ?? '').replace(/\s+/g, '');
|
||||
if (!encoded || encoded.length % 4 === 1 || !BASE64_PATTERN.test(encoded)) {
|
||||
throw new InputAttachmentError('roster_file_base64_invalid', '名单附件内容不是合法 Base64。');
|
||||
}
|
||||
const normalizedBase64 = encoded.replace(/-/g, '+').replace(/_/g, '/');
|
||||
const content = Buffer.from(normalizedBase64, 'base64');
|
||||
validateAttachmentBytes(content, maxBytes, declaredSize, declaredSha256);
|
||||
return {
|
||||
fileName,
|
||||
contentType: normalizeContentType(value.content_type),
|
||||
content,
|
||||
declaredSize,
|
||||
declaredSha256,
|
||||
source
|
||||
};
|
||||
}
|
||||
|
||||
function ipv4Number(address: string): number | null {
|
||||
const parts = address.split('.').map(Number);
|
||||
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return null;
|
||||
return (((parts[0] * 256 + parts[1]) * 256 + parts[2]) * 256 + parts[3]) >>> 0;
|
||||
}
|
||||
|
||||
function ipv4InCidr(address: string, network: string, prefix: number): boolean {
|
||||
const value = ipv4Number(address);
|
||||
const base = ipv4Number(network);
|
||||
if (value === null || base === null) return false;
|
||||
const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0;
|
||||
return (value & mask) === (base & mask);
|
||||
}
|
||||
|
||||
export function isPrivateOrReservedIp(address: string): boolean {
|
||||
const family = isIP(address);
|
||||
if (family === 4) {
|
||||
return [
|
||||
['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8],
|
||||
['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24],
|
||||
['192.168.0.0', 16], ['198.18.0.0', 15], ['198.51.100.0', 24], ['203.0.113.0', 24],
|
||||
['224.0.0.0', 4], ['240.0.0.0', 4]
|
||||
].some(([network, prefix]) => ipv4InCidr(address, String(network), Number(prefix)));
|
||||
}
|
||||
if (family !== 6) return true;
|
||||
const normalized = address.toLowerCase().split('%')[0];
|
||||
if (normalized === '::' || normalized === '::1') return true;
|
||||
const mapped = /^(?:::ffff:)?(\d+\.\d+\.\d+\.\d+)$/.exec(normalized);
|
||||
if (mapped) return isPrivateOrReservedIp(mapped[1]);
|
||||
return /^(?:fc|fd)/.test(normalized)
|
||||
|| /^fe[89ab]/.test(normalized)
|
||||
|| /^ff/.test(normalized)
|
||||
|| normalized.startsWith('2001:db8:');
|
||||
}
|
||||
|
||||
export function validateAgentBusAttachmentUrl(value: unknown): URL {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(String(value ?? ''));
|
||||
} catch {
|
||||
throw new InputAttachmentError('roster_attachment_url_invalid', '名单附件 URL 无效。');
|
||||
}
|
||||
if (url.protocol !== 'https:' || url.username || url.password) {
|
||||
throw new InputAttachmentError('roster_attachment_url_unsafe', '名单附件必须使用不含用户名密码的 HTTPS URL。');
|
||||
}
|
||||
const hostname = url.hostname.replace(/^\[|\]$/g, '');
|
||||
if (!hostname || hostname.toLowerCase() === 'localhost') {
|
||||
throw new InputAttachmentError('roster_attachment_url_unsafe', '名单附件 URL 指向了不允许的地址。');
|
||||
}
|
||||
if (isIP(hostname) && isPrivateOrReservedIp(hostname)) {
|
||||
throw new InputAttachmentError('roster_attachment_url_unsafe', '名单附件 URL 指向了不允许的地址。');
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
export function parseAgentBusInputAttachment(
|
||||
value: unknown,
|
||||
maxBytes: number
|
||||
): AgentBusInputAttachmentReference {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InputAttachmentError('roster_attachment_metadata_invalid', '名单附件元数据无效。');
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return {
|
||||
name: normalizeInputAttachmentFileName(record.name),
|
||||
contentType: normalizeContentType(record.content_type),
|
||||
size: normalizeDeclaredSize(record.size, maxBytes),
|
||||
sha256: normalizeDeclaredSha256(record.sha256),
|
||||
url: validateAgentBusAttachmentUrl(record.url).toString()
|
||||
};
|
||||
}
|
||||
|
||||
async function publicAddresses(hostname: string): Promise<Array<{ address: string; family: number }>> {
|
||||
if (isIP(hostname)) return [{ address: hostname, family: isIP(hostname) }];
|
||||
let addresses: Array<{ address: string; family: number }>;
|
||||
try {
|
||||
addresses = await dnsLookup(hostname, { all: true, verbatim: true });
|
||||
} catch {
|
||||
throw new InputAttachmentError('roster_attachment_host_unresolved', '名单附件地址无法解析。');
|
||||
}
|
||||
if (!addresses.length || addresses.some((item) => isPrivateOrReservedIp(item.address))) {
|
||||
throw new InputAttachmentError('roster_attachment_url_unsafe', '名单附件地址解析到了不允许的网络。');
|
||||
}
|
||||
return addresses;
|
||||
}
|
||||
|
||||
async function downloadPinnedHttps(
|
||||
url: URL,
|
||||
address: { address: string; family: number },
|
||||
maxBytes: number
|
||||
): Promise<{ statusCode: number; location?: string; contentType: string; content: Buffer }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = httpsRequest(url, {
|
||||
method: 'GET',
|
||||
headers: { Accept: 'application/vnd.ms-excel, application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, application/octet-stream' },
|
||||
timeout: DOWNLOAD_TIMEOUT_MS,
|
||||
lookup: ((_hostname: string, _options: unknown, callback: (error: NodeJS.ErrnoException | null, address: string, family: number) => void) => {
|
||||
callback(null, address.address, address.family);
|
||||
}) as never
|
||||
}, (response) => {
|
||||
const statusCode = Number(response.statusCode || 0);
|
||||
const location = Array.isArray(response.headers.location) ? response.headers.location[0] : response.headers.location;
|
||||
if (statusCode >= 300 && statusCode < 400) {
|
||||
response.resume();
|
||||
resolve({ statusCode, location, contentType: '', content: Buffer.alloc(0) });
|
||||
return;
|
||||
}
|
||||
if (statusCode !== 200) {
|
||||
response.resume();
|
||||
reject(new InputAttachmentError('roster_attachment_download_failed', '名单附件下载失败。', { status_code: statusCode }));
|
||||
return;
|
||||
}
|
||||
const declaredLength = Number(response.headers['content-length']);
|
||||
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
||||
response.destroy();
|
||||
reject(new InputAttachmentError('roster_file_too_large', '名单附件超过大小限制。', { max_bytes: maxBytes }));
|
||||
return;
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
response.on('data', (chunk: Buffer | string) => {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
total += buffer.byteLength;
|
||||
if (total > maxBytes) {
|
||||
response.destroy(new InputAttachmentError('roster_file_too_large', '名单附件超过大小限制。', { max_bytes: maxBytes }));
|
||||
return;
|
||||
}
|
||||
chunks.push(buffer);
|
||||
});
|
||||
response.once('end', () => resolve({
|
||||
statusCode,
|
||||
contentType: normalizeContentType(response.headers['content-type']),
|
||||
content: Buffer.concat(chunks, total)
|
||||
}));
|
||||
response.once('error', reject);
|
||||
});
|
||||
request.once('timeout', () => request.destroy(new InputAttachmentError('roster_attachment_download_timeout', '名单附件下载超时。')));
|
||||
request.once('error', reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
export async function downloadAgentBusInputAttachment(
|
||||
reference: AgentBusInputAttachmentReference,
|
||||
maxBytes: number
|
||||
): Promise<TaskInputAttachmentInput> {
|
||||
let url = validateAgentBusAttachmentUrl(reference.url);
|
||||
for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects += 1) {
|
||||
const addresses = await publicAddresses(url.hostname.replace(/^\[|\]$/g, ''));
|
||||
const response = await downloadPinnedHttps(url, addresses[0], maxBytes);
|
||||
if (response.statusCode >= 300 && response.statusCode < 400) {
|
||||
if (!response.location || redirects === MAX_REDIRECTS) {
|
||||
throw new InputAttachmentError('roster_attachment_redirect_invalid', '名单附件重定向无效或次数过多。');
|
||||
}
|
||||
url = validateAgentBusAttachmentUrl(new URL(response.location, url).toString());
|
||||
continue;
|
||||
}
|
||||
validateAttachmentBytes(response.content, maxBytes, reference.size, reference.sha256);
|
||||
return {
|
||||
fileName: reference.name,
|
||||
contentType: response.contentType === 'application/octet-stream' ? reference.contentType : response.contentType,
|
||||
content: response.content,
|
||||
declaredSize: reference.size,
|
||||
declaredSha256: reference.sha256,
|
||||
source: 'agentbus'
|
||||
};
|
||||
}
|
||||
throw new InputAttachmentError('roster_attachment_download_failed', '名单附件下载失败。');
|
||||
}
|
||||
Reference in new issue
Block a user