import { extname } from 'node:path'; import ExcelJS from 'exceljs'; import { fromBufferPromise, type Entry, type ZipFile } from 'yauzl'; import type { DocumentConversionErrorCode } from './document-converter.js'; const XLSX_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04]); const XLS_MAGIC = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]); const DEFAULT_MAX_INPUT_BYTES = 10 * 1024 * 1024; const DEFAULT_MAX_UNCOMPRESSED_BYTES = 64 * 1024 * 1024; const DEFAULT_MAX_ARCHIVE_ENTRIES = 2_048; const DEFAULT_MAX_DATA_ROWS = 5_000; const ABSOLUTE_MAX_DATA_ROWS = 5_000; const MAX_PASSENGER_SEQUENCE = 5_000; const MAX_HEADER_ROW = 100; const EXPECTED_HEADER_ROW = 2; const SOURCE_COLUMN_COUNT = 14; export const PASSENGER_ROSTER_WORKBOOK_VERSION = 'ltjt-passenger-roster-workbook-v1.1.0'; const SOURCE_HEADERS = [ '序号', '姓名', '英文姓名', '性别', '身份证号码', '出生日期', '年龄', '出生地', '护照号码', '签发地', '签发日期', '有效期', '电话', '备注' ] as const; export const CANONICAL_PASSENGER_HEADERS = [ '序号', '姓名', 'NAME', '性别', '出生日期', '出生地', '证件类型', '证件号码', '签发地', '签发日', '有效期', '电话', '备注' ] as const; export type PassengerRosterWorkbookErrorCode = | 'roster_workbook_empty' | 'roster_workbook_too_large' | 'roster_workbook_unsupported_format' | 'roster_workbook_format_mismatch' | 'roster_workbook_converter_unavailable' | 'roster_workbook_conversion_failed' | 'roster_workbook_invalid_xlsx' | 'roster_workbook_archive_limit' | 'roster_workbook_unsafe_content' | 'roster_workbook_ambiguous_sheet' | 'roster_workbook_hidden_data' | 'roster_workbook_header_not_found' | 'roster_workbook_ambiguous_header' | 'roster_workbook_header_row_invalid' | 'roster_workbook_formula_not_allowed' | 'roster_workbook_formula_result_missing' | 'roster_workbook_unexpected_column' | 'roster_workbook_non_contiguous_rows' | 'roster_workbook_row_limit' | 'roster_workbook_no_data' | 'roster_workbook_invalid_sequence' | 'roster_workbook_sequence_limit' | 'roster_workbook_duplicate_sequence' | 'roster_workbook_identity_card_not_supported' | 'roster_workbook_invalid_date' | 'roster_workbook_invalid_identifier' | 'roster_workbook_invalid_cell'; const ERROR_MESSAGES: Record = { roster_workbook_empty: '名单工作簿为空', roster_workbook_too_large: '名单工作簿超过大小限制', roster_workbook_unsupported_format: '名单附件格式不受支持', roster_workbook_format_mismatch: '名单附件扩展名、类型或文件内容不匹配', roster_workbook_converter_unavailable: '旧版名单工作簿转换器不可用', roster_workbook_conversion_failed: '旧版名单工作簿转换失败', roster_workbook_invalid_xlsx: '名单工作簿不是有效的 XLSX', roster_workbook_archive_limit: '名单工作簿解压规模超过限制', roster_workbook_unsafe_content: '名单工作簿包含不允许的宏、外部链接或嵌入内容', roster_workbook_ambiguous_sheet: '名单工作簿必须且只能包含一个工作表', roster_workbook_hidden_data: '名单工作簿包含隐藏的非空数据', roster_workbook_header_not_found: '未找到完整的 14 列名单表头', roster_workbook_ambiguous_header: '名单工作簿包含多个可识别的名单表头', roster_workbook_header_row_invalid: '名单工作簿第 2 行必须是完整的 14 列名单表头', roster_workbook_formula_not_allowed: '名单工作簿包含不允许的公式', roster_workbook_formula_result_missing: '名单工作簿公式没有可验证的缓存结果', roster_workbook_unexpected_column: '名单数据包含 14 列之外的内容', roster_workbook_non_contiguous_rows: '名单数据行不连续', roster_workbook_row_limit: '名单数据行数超过技术限制', roster_workbook_no_data: '名单工作簿没有数据行', roster_workbook_invalid_sequence: '名单序号必须是正整数', roster_workbook_sequence_limit: '名单序号超过技术安全上限', roster_workbook_duplicate_sequence: '名单序号不能重复', roster_workbook_identity_card_not_supported: '当前名单导入不支持非空身份证号码', roster_workbook_invalid_date: '名单日期无效', roster_workbook_invalid_identifier: '名单证件号或电话无法安全转换为文本', roster_workbook_invalid_cell: '名单单元格值无效' }; export class PassengerRosterWorkbookError extends Error { readonly code: PassengerRosterWorkbookErrorCode; readonly row?: number; readonly column?: number; readonly conversionErrorCode?: DocumentConversionErrorCode; constructor( code: PassengerRosterWorkbookErrorCode, location: { row?: number; column?: number; conversionErrorCode?: DocumentConversionErrorCode } = {} ) { super(ERROR_MESSAGES[code]); this.name = 'PassengerRosterWorkbookError'; this.code = code; this.row = location.row; this.column = location.column; this.conversionErrorCode = location.conversionErrorCode; } } export interface PassengerRosterWorkbookInput { content: Buffer; fileName: string; contentType: string; } /** * The caller must run legacy conversion in an isolated, macro-disabled * process and must reject converter fallback to the original source bytes. * This normalizer independently verifies that the callback returned XLSX. */ export type LegacyXlsConverter = (input: PassengerRosterWorkbookInput) => Promise; export interface PassengerRosterWorkbookOptions { convertLegacyXls?: LegacyXlsConverter; maxInputBytes?: number; maxUncompressedBytes?: number; maxArchiveEntries?: number; maxDataRows?: number; } export interface NormalizedPassengerRosterWorkbook { canonicalTsv: string; rowCount: number; sourceFormat: 'xls' | 'xlsx'; worksheetName: string; headerRow: number; } type ScalarCellValue = string | number | boolean | Date | null; function fail( code: PassengerRosterWorkbookErrorCode, location: { row?: number; column?: number; conversionErrorCode?: DocumentConversionErrorCode } = {} ): never { throw new PassengerRosterWorkbookError(code, location); } function startsWithMagic(content: Buffer, magic: Buffer): boolean { return content.byteLength >= magic.byteLength && content.subarray(0, magic.byteLength).equals(magic); } function normalizedMimeType(contentType: string): string { return contentType.toLowerCase().split(';', 1)[0].trim(); } function sourceFormat(input: PassengerRosterWorkbookInput): 'xls' | 'xlsx' { const extension = extname(input.fileName).toLowerCase(); const mimeType = normalizedMimeType(input.contentType); const xlsxMagic = startsWithMagic(input.content, XLSX_MAGIC); const xlsMagic = startsWithMagic(input.content, XLS_MAGIC); const genericMime = mimeType === '' || mimeType === 'application/octet-stream' || mimeType === 'application/zip'; const xlsxMime = mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; const xlsMime = [ 'application/vnd.ms-excel', 'application/msexcel', 'application/x-msexcel', 'application/xls', 'application/vnd.ms-office' ].includes(mimeType); if (extension !== '.xls' && extension !== '.xlsx') { fail('roster_workbook_unsupported_format'); } if (extension === '.xlsx') { if (!xlsxMagic || xlsMagic || (!genericMime && !xlsxMime)) { fail('roster_workbook_format_mismatch'); } return 'xlsx'; } if (!xlsMagic || xlsxMagic || (!genericMime && !xlsMime)) { fail('roster_workbook_format_mismatch'); } return 'xls'; } function legacyWorkbookHasMacroMarkers(content: Buffer): boolean { const markers = ['_VBA_PROJECT_CUR', 'PROJECTwm', 'VBA/dir', 'VBA\\dir', 'Macros']; return markers.some((marker) => ( content.indexOf(Buffer.from(marker, 'latin1')) >= 0 || content.indexOf(Buffer.from(marker, 'utf16le')) >= 0 )); } const UNSAFE_ARCHIVE_PATH = /(?:^|\/)(?:vbaProject\.bin|macrosheets\/|activex\/|embeddings\/|externalLinks\/|connections\.xml$|queryTables\/|webextensions\/|model\/)/i; const RELATIONSHIP_OR_CONTENT_TYPE_PATH = /(?:\.rels|\[Content_Types\]\.xml)$/i; const EXTERNAL_RELATIONSHIP_PATTERN = /\bTargetMode\s*=\s*["']External["']/i; const MACRO_CONTENT_TYPE_PATTERN = /(?:macroEnabled|vbaProject|activeX|oleObject)/i; async function readArchiveEntryText(zipFile: ZipFile, entry: Entry): Promise { const stream = await zipFile.openReadStreamPromise(entry); const chunks: Buffer[] = []; for await (const chunk of stream) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); } return Buffer.concat(chunks).toString('utf8'); } async function inspectXlsxArchive( content: Buffer, options: Required> ): Promise { let zipFile: ZipFile; try { zipFile = await fromBufferPromise(content, { lazyEntries: true, decodeStrings: true, validateEntrySizes: true, strictFileNames: true }); } catch { fail('roster_workbook_invalid_xlsx'); } let entryCount = 0; let uncompressedBytes = 0; try { for await (const entry of zipFile.eachEntry()) { entryCount += 1; uncompressedBytes += entry.uncompressedSize; if ( entryCount > options.maxArchiveEntries || entry.uncompressedSize > options.maxUncompressedBytes || uncompressedBytes > options.maxUncompressedBytes ) { fail('roster_workbook_archive_limit'); } if (entry.isEncrypted() || !entry.canDecodeFileData()) { fail('roster_workbook_unsafe_content'); } const normalizedPath = entry.fileName.replace(/\\/g, '/'); if (UNSAFE_ARCHIVE_PATH.test(normalizedPath)) { fail('roster_workbook_unsafe_content'); } if (RELATIONSHIP_OR_CONTENT_TYPE_PATH.test(normalizedPath) && entry.uncompressedSize > 0) { const source = await readArchiveEntryText(zipFile, entry); if ( EXTERNAL_RELATIONSHIP_PATTERN.test(source) || (normalizedPath.endsWith('[Content_Types].xml') && MACRO_CONTENT_TYPE_PATTERN.test(source)) ) { fail('roster_workbook_unsafe_content'); } } } } catch (error) { if (error instanceof PassengerRosterWorkbookError) throw error; fail('roster_workbook_invalid_xlsx'); } finally { zipFile.close(); } } function formulaText(cell: ExcelJS.Cell): string | undefined { if (cell.type !== ExcelJS.ValueType.Formula) return undefined; const formula = cell.formula; return typeof formula === 'string' ? formula : undefined; } function containsExternalFormulaReference(formula: string): boolean { return /(?:\[[^\]]+\]|https?:|ftp:|file:|\\\\|WEBSERVICE\s*\(|HYPERLINK\s*\(|DDE\s*\()/i.test(formula); } function normalizedFormula(formula: string): string { return formula .replace(/^=/, '') .replace(/\$/g, '') .replace(/;/g, ',') .replace(/\s+/g, '') .toUpperCase(); } function cellRow(cell: ExcelJS.Cell): number { return cell.fullAddress.row; } function cellColumn(cell: ExcelJS.Cell): number { return cell.fullAddress.col; } function formulaAllowed(cell: ExcelJS.Cell, headerRow: number): boolean { const row = cellRow(cell); const column = cellColumn(cell); if (row <= headerRow) return false; const formula = normalizedFormula(cell.formula); if (column === 7) { return formula === `DATEDIF(F${row},TODAY(),"Y")`; } if (column === 12) { return formula === `EDATE(K${row},10*12)-1`; } return false; } function scalarValue(cell: ExcelJS.Cell, allowFormulaResult = false): ScalarCellValue { if (cell.type === ExcelJS.ValueType.Formula) { if (!allowFormulaResult) fail('roster_workbook_formula_not_allowed', { row: cellRow(cell), column: cellColumn(cell) }); const result = cell.result; if (result === undefined || result === null) { fail('roster_workbook_formula_result_missing', { row: cellRow(cell), column: cellColumn(cell) }); } if (typeof result === 'object' && !(result instanceof Date)) { fail('roster_workbook_invalid_cell', { row: cellRow(cell), column: cellColumn(cell) }); } return result; } const value = cell.value; if (value === undefined || value === null) return null; if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean' || value instanceof Date) { return value; } if ('hyperlink' in value) { fail('roster_workbook_unsafe_content', { row: cellRow(cell), column: cellColumn(cell) }); } if ('richText' in value) { return value.richText.map((part) => part.text).join(''); } fail('roster_workbook_invalid_cell', { row: cellRow(cell), column: cellColumn(cell) }); } function normalizedText(value: ScalarCellValue): string { if (value === null) return ''; if (value instanceof Date) return value.toISOString(); return String(value) .replace(/[\t\r\n\u3000]+/g, ' ') .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '') .replace(/\s+/g, ' ') .trim(); } function headerText(cell: ExcelJS.Cell): string { if (cell.type === ExcelJS.ValueType.Formula) return ''; return normalizedText(scalarValue(cell)); } function cellHasData(cell: ExcelJS.Cell): boolean { if (cell.type === ExcelJS.ValueType.Formula) return true; return normalizedText(scalarValue(cell)) !== ''; } function rowHasData(row: ExcelJS.Row, maxColumn = SOURCE_COLUMN_COUNT): boolean { for (let column = 1; column <= maxColumn; column += 1) { if (cellHasData(row.getCell(column))) return true; } return false; } function validateFormulasAndHiddenData(sheet: ExcelJS.Worksheet, headerRow: number): void { sheet.eachRow({ includeEmpty: false }, (row, rowNumber) => { let hasData = false; row.eachCell({ includeEmpty: false }, (cell, columnNumber) => { if (cell.type === ExcelJS.ValueType.Merge && cell.master.address !== cell.address) return; const formula = formulaText(cell); if (formula) { if (containsExternalFormulaReference(formula)) { fail('roster_workbook_unsafe_content', { row: rowNumber, column: columnNumber }); } if (!formulaAllowed(cell, headerRow)) { fail('roster_workbook_formula_not_allowed', { row: rowNumber, column: columnNumber }); } } if (cell.type === ExcelJS.ValueType.Hyperlink || cell.isHyperlink) { fail('roster_workbook_unsafe_content', { row: rowNumber, column: columnNumber }); } if (cellHasData(cell)) hasData = true; if (columnNumber > SOURCE_COLUMN_COUNT && rowNumber >= headerRow) { fail('roster_workbook_unexpected_column', { row: rowNumber, column: columnNumber }); } }); if (row.hidden && hasData) fail('roster_workbook_hidden_data', { row: rowNumber }); }); for (let columnNumber = 1; columnNumber <= sheet.columnCount; columnNumber += 1) { const column = sheet.getColumn(columnNumber); if (!column.hidden) continue; let hasData = false; column.eachCell({ includeEmpty: false }, (cell) => { if (cellHasData(cell)) hasData = true; }); if (hasData) fail('roster_workbook_hidden_data', { column: columnNumber }); } } function findHeaderRow(sheet: ExcelJS.Worksheet): number { const candidates: number[] = []; const lastCandidate = Math.min(sheet.rowCount, MAX_HEADER_ROW); for (let rowNumber = 1; rowNumber <= lastCandidate; rowNumber += 1) { const row = sheet.getRow(rowNumber); const values = SOURCE_HEADERS.map((_header, index) => headerText(row.getCell(index + 1))); if (values.every((value, index) => value === SOURCE_HEADERS[index])) { candidates.push(rowNumber); } } if (candidates.length === 0) fail('roster_workbook_header_not_found'); if (candidates.length !== 1) fail('roster_workbook_ambiguous_header'); if (candidates[0] !== EXPECTED_HEADER_ROW) { fail('roster_workbook_header_row_invalid', { row: candidates[0] }); } return candidates[0]; } function excelSerialToDate(value: number, date1904: boolean): Date | undefined { if (!Number.isFinite(value) || value < 0) return undefined; const millisecondsPerDay = 86_400_000; const epoch = date1904 ? Date.UTC(1904, 0, 1) : Date.UTC(1899, 11, 30); const result = new Date(epoch + Math.floor(value) * millisecondsPerDay); return Number.isNaN(result.getTime()) ? undefined : result; } function canonicalDate(cell: ExcelJS.Cell, date1904: boolean, formulaResult = false): string { const value = scalarValue(cell, formulaResult); if (value === null || value === '') return ''; let date: Date | undefined; if (value instanceof Date) { date = value; } else if (typeof value === 'number') { date = excelSerialToDate(value, date1904); } else if (typeof value === 'string') { const match = value.trim().match(/^(\d{4})[-/.\u5e74](\d{1,2})[-/.\u6708](\d{1,2})(?:\u65e5)?$/); if (match) { date = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]))); if ( date.getUTCFullYear() !== Number(match[1]) || date.getUTCMonth() !== Number(match[2]) - 1 || date.getUTCDate() !== Number(match[3]) ) { date = undefined; } } } if (!date || Number.isNaN(date.getTime())) { fail('roster_workbook_invalid_date', { row: cellRow(cell), column: cellColumn(cell) }); } const year = date.getUTCFullYear(); if (year < 1900 || year > 2200) { fail('roster_workbook_invalid_date', { row: cellRow(cell), column: cellColumn(cell) }); } return `${String(year).padStart(4, '0')}-${String(date.getUTCMonth() + 1).padStart(2, '0')}-${String(date.getUTCDate()).padStart(2, '0')}`; } function canonicalIdentifier(cell: ExcelJS.Cell): string { const value = scalarValue(cell); if (value === null || value === '') return ''; if (typeof value === 'number') { if (!Number.isSafeInteger(value) || value < 0) { fail('roster_workbook_invalid_identifier', { row: cellRow(cell), column: cellColumn(cell) }); } return String(value); } if (typeof value !== 'string') { fail('roster_workbook_invalid_identifier', { row: cellRow(cell), column: cellColumn(cell) }); } return normalizedText(value); } function canonicalSequence(cell: ExcelJS.Cell): number { const value = scalarValue(cell); let sequence: number; if (typeof value === 'number') { sequence = value; } else if (typeof value === 'string' && /^[1-9]\d*$/.test(value.trim())) { sequence = Number(value.trim()); } else { fail('roster_workbook_invalid_sequence', { row: cellRow(cell), column: cellColumn(cell) }); } if (!Number.isSafeInteger(sequence) || sequence <= 0) { fail('roster_workbook_invalid_sequence', { row: cellRow(cell), column: cellColumn(cell) }); } if (sequence > MAX_PASSENGER_SEQUENCE) { fail('roster_workbook_sequence_limit', { row: cellRow(cell), column: cellColumn(cell) }); } return sequence; } function canonicalTextCell(cell: ExcelJS.Cell): string { return normalizedText(scalarValue(cell)); } function dataRows(sheet: ExcelJS.Worksheet, headerRow: number, maxDataRows: number): ExcelJS.Row[] { const rows: ExcelJS.Row[] = []; let gapAfterData = false; for (let rowNumber = headerRow + 1; rowNumber <= sheet.rowCount; rowNumber += 1) { const row = sheet.getRow(rowNumber); const hasData = rowHasData(row); if (!hasData) { if (rows.length > 0) gapAfterData = true; continue; } if (gapAfterData) fail('roster_workbook_non_contiguous_rows', { row: rowNumber }); rows.push(row); if (rows.length > maxDataRows) fail('roster_workbook_row_limit'); } if (rows.length === 0) fail('roster_workbook_no_data'); return rows; } function canonicalRow(row: ExcelJS.Row, date1904: boolean, seenSequences: Set): string[] { const sequence = canonicalSequence(row.getCell(1)); if (seenSequences.has(sequence)) { fail('roster_workbook_duplicate_sequence', { row: row.number, column: 1 }); } seenSequences.add(sequence); if (canonicalIdentifier(row.getCell(5)) !== '') { fail('roster_workbook_identity_card_not_supported', { row: row.number, column: 5 }); } const englishName = canonicalTextCell(row.getCell(3)).replace(/,/g, ','); return [ String(sequence), canonicalTextCell(row.getCell(2)), englishName, canonicalTextCell(row.getCell(4)), canonicalDate(row.getCell(6), date1904), canonicalTextCell(row.getCell(8)), '护照', canonicalIdentifier(row.getCell(9)), canonicalTextCell(row.getCell(10)), canonicalDate(row.getCell(11), date1904), canonicalDate(row.getCell(12), date1904, row.getCell(12).type === ExcelJS.ValueType.Formula), canonicalIdentifier(row.getCell(13)), canonicalTextCell(row.getCell(14)) ]; } export async function normalizePassengerRosterWorkbook( input: PassengerRosterWorkbookInput, options: PassengerRosterWorkbookOptions = {} ): Promise { const maxInputBytes = options.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; const maxUncompressedBytes = options.maxUncompressedBytes ?? DEFAULT_MAX_UNCOMPRESSED_BYTES; const maxArchiveEntries = options.maxArchiveEntries ?? DEFAULT_MAX_ARCHIVE_ENTRIES; const requestedMaxDataRows = options.maxDataRows ?? DEFAULT_MAX_DATA_ROWS; const maxDataRows = Math.min(requestedMaxDataRows, ABSOLUTE_MAX_DATA_ROWS); if (!Number.isSafeInteger(maxInputBytes) || maxInputBytes <= 0) fail('roster_workbook_too_large'); if (!Number.isSafeInteger(maxUncompressedBytes) || maxUncompressedBytes <= 0) fail('roster_workbook_archive_limit'); if (!Number.isSafeInteger(maxArchiveEntries) || maxArchiveEntries <= 0) fail('roster_workbook_archive_limit'); if (!Number.isSafeInteger(requestedMaxDataRows) || requestedMaxDataRows <= 0) fail('roster_workbook_row_limit'); if (input.content.byteLength === 0) fail('roster_workbook_empty'); if (input.content.byteLength > maxInputBytes) fail('roster_workbook_too_large'); const format = sourceFormat(input); let xlsxContent = input.content; if (format === 'xls') { if (legacyWorkbookHasMacroMarkers(input.content)) fail('roster_workbook_unsafe_content'); if (!options.convertLegacyXls) fail('roster_workbook_converter_unavailable'); try { xlsxContent = await options.convertLegacyXls(input); } catch (error) { if (error instanceof PassengerRosterWorkbookError) throw error; fail('roster_workbook_conversion_failed'); } if ( !Buffer.isBuffer(xlsxContent) || xlsxContent.byteLength === 0 || xlsxContent.byteLength > maxInputBytes || !startsWithMagic(xlsxContent, XLSX_MAGIC) ) { fail('roster_workbook_conversion_failed'); } } await inspectXlsxArchive(xlsxContent, { maxUncompressedBytes, maxArchiveEntries }); const workbook = new ExcelJS.Workbook(); try { const excelJsBuffer = xlsxContent as unknown as Parameters[0]; await workbook.xlsx.load(excelJsBuffer); } catch { fail('roster_workbook_invalid_xlsx'); } if (workbook.worksheets.length !== 1) fail('roster_workbook_ambiguous_sheet'); const sheet = workbook.worksheets[0]; if (sheet.state !== 'visible') fail('roster_workbook_hidden_data'); const headerRow = findHeaderRow(sheet); validateFormulasAndHiddenData(sheet, headerRow); const rows = dataRows(sheet, headerRow, maxDataRows); const date1904 = workbook.properties.date1904 === true; const seenSequences = new Set(); const canonicalRows = rows.map((row) => canonicalRow(row, date1904, seenSequences)); return { canonicalTsv: [CANONICAL_PASSENGER_HEADERS, ...canonicalRows] .map((row) => row.join('\t')) .join('\n'), rowCount: canonicalRows.length, sourceFormat: format, worksheetName: sheet.name, headerRow }; }