feat: ship deterministic parser and lifecycle release
This commit is contained in:
inman committed
1 parent
208c434b31
commit
7b5d855b09
176 files changed
+20248
-1362
No files matched your search
@@ -0,0 +1,595 @@
|
||||
import { extname } from 'node:path';
|
||||
import ExcelJS from 'exceljs';
|
||||
import { fromBufferPromise, type Entry, type ZipFile } from 'yauzl';
|
||||
|
||||
const XLSX_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
|
||||
const XLS_MAGIC = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
|
||||
const DEFAULT_MAX_INPUT_BYTES = 10 * 1024 * 1024;
|
||||
const DEFAULT_MAX_UNCOMPRESSED_BYTES = 64 * 1024 * 1024;
|
||||
const DEFAULT_MAX_ARCHIVE_ENTRIES = 2_048;
|
||||
const DEFAULT_MAX_DATA_ROWS = 5_000;
|
||||
const ABSOLUTE_MAX_DATA_ROWS = 5_000;
|
||||
const MAX_PASSENGER_SEQUENCE = 5_000;
|
||||
const MAX_HEADER_ROW = 100;
|
||||
const EXPECTED_HEADER_ROW = 2;
|
||||
const SOURCE_COLUMN_COUNT = 14;
|
||||
|
||||
export const PASSENGER_ROSTER_WORKBOOK_VERSION = 'ltjt-passenger-roster-workbook-v1.1.0';
|
||||
|
||||
const SOURCE_HEADERS = [
|
||||
'序号', '姓名', '英文姓名', '性别', '身份证号码', '出生日期', '年龄', '出生地',
|
||||
'护照号码', '签发地', '签发日期', '有效期', '电话', '备注'
|
||||
] as const;
|
||||
|
||||
export const CANONICAL_PASSENGER_HEADERS = [
|
||||
'序号', '姓名', 'NAME', '性别', '出生日期', '出生地', '证件类型', '证件号码',
|
||||
'签发地', '签发日', '有效期', '电话', '备注'
|
||||
] as const;
|
||||
|
||||
export type PassengerRosterWorkbookErrorCode =
|
||||
| 'roster_workbook_empty'
|
||||
| 'roster_workbook_too_large'
|
||||
| 'roster_workbook_unsupported_format'
|
||||
| 'roster_workbook_format_mismatch'
|
||||
| 'roster_workbook_converter_unavailable'
|
||||
| 'roster_workbook_conversion_failed'
|
||||
| 'roster_workbook_invalid_xlsx'
|
||||
| 'roster_workbook_archive_limit'
|
||||
| 'roster_workbook_unsafe_content'
|
||||
| 'roster_workbook_ambiguous_sheet'
|
||||
| 'roster_workbook_hidden_data'
|
||||
| 'roster_workbook_header_not_found'
|
||||
| 'roster_workbook_ambiguous_header'
|
||||
| 'roster_workbook_header_row_invalid'
|
||||
| 'roster_workbook_formula_not_allowed'
|
||||
| 'roster_workbook_formula_result_missing'
|
||||
| 'roster_workbook_unexpected_column'
|
||||
| 'roster_workbook_non_contiguous_rows'
|
||||
| 'roster_workbook_row_limit'
|
||||
| 'roster_workbook_no_data'
|
||||
| 'roster_workbook_invalid_sequence'
|
||||
| 'roster_workbook_sequence_limit'
|
||||
| 'roster_workbook_duplicate_sequence'
|
||||
| 'roster_workbook_identity_card_not_supported'
|
||||
| 'roster_workbook_invalid_date'
|
||||
| 'roster_workbook_invalid_identifier'
|
||||
| 'roster_workbook_invalid_cell';
|
||||
|
||||
const ERROR_MESSAGES: Record<PassengerRosterWorkbookErrorCode, string> = {
|
||||
roster_workbook_empty: '名单工作簿为空',
|
||||
roster_workbook_too_large: '名单工作簿超过大小限制',
|
||||
roster_workbook_unsupported_format: '名单附件格式不受支持',
|
||||
roster_workbook_format_mismatch: '名单附件扩展名、类型或文件内容不匹配',
|
||||
roster_workbook_converter_unavailable: '旧版名单工作簿转换器不可用',
|
||||
roster_workbook_conversion_failed: '旧版名单工作簿转换失败',
|
||||
roster_workbook_invalid_xlsx: '名单工作簿不是有效的 XLSX',
|
||||
roster_workbook_archive_limit: '名单工作簿解压规模超过限制',
|
||||
roster_workbook_unsafe_content: '名单工作簿包含不允许的宏、外部链接或嵌入内容',
|
||||
roster_workbook_ambiguous_sheet: '名单工作簿必须且只能包含一个工作表',
|
||||
roster_workbook_hidden_data: '名单工作簿包含隐藏的非空数据',
|
||||
roster_workbook_header_not_found: '未找到完整的 14 列名单表头',
|
||||
roster_workbook_ambiguous_header: '名单工作簿包含多个可识别的名单表头',
|
||||
roster_workbook_header_row_invalid: '名单工作簿第 2 行必须是完整的 14 列名单表头',
|
||||
roster_workbook_formula_not_allowed: '名单工作簿包含不允许的公式',
|
||||
roster_workbook_formula_result_missing: '名单工作簿公式没有可验证的缓存结果',
|
||||
roster_workbook_unexpected_column: '名单数据包含 14 列之外的内容',
|
||||
roster_workbook_non_contiguous_rows: '名单数据行不连续',
|
||||
roster_workbook_row_limit: '名单数据行数超过技术限制',
|
||||
roster_workbook_no_data: '名单工作簿没有数据行',
|
||||
roster_workbook_invalid_sequence: '名单序号必须是正整数',
|
||||
roster_workbook_sequence_limit: '名单序号超过技术安全上限',
|
||||
roster_workbook_duplicate_sequence: '名单序号不能重复',
|
||||
roster_workbook_identity_card_not_supported: '当前名单导入不支持非空身份证号码',
|
||||
roster_workbook_invalid_date: '名单日期无效',
|
||||
roster_workbook_invalid_identifier: '名单证件号或电话无法安全转换为文本',
|
||||
roster_workbook_invalid_cell: '名单单元格值无效'
|
||||
};
|
||||
|
||||
export class PassengerRosterWorkbookError extends Error {
|
||||
readonly code: PassengerRosterWorkbookErrorCode;
|
||||
readonly row?: number;
|
||||
readonly column?: number;
|
||||
|
||||
constructor(code: PassengerRosterWorkbookErrorCode, location: { row?: number; column?: number } = {}) {
|
||||
super(ERROR_MESSAGES[code]);
|
||||
this.name = 'PassengerRosterWorkbookError';
|
||||
this.code = code;
|
||||
this.row = location.row;
|
||||
this.column = location.column;
|
||||
}
|
||||
}
|
||||
|
||||
export interface PassengerRosterWorkbookInput {
|
||||
content: Buffer;
|
||||
fileName: string;
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The caller must run legacy conversion in an isolated, macro-disabled
|
||||
* process and must reject converter fallback to the original source bytes.
|
||||
* This normalizer independently verifies that the callback returned XLSX.
|
||||
*/
|
||||
export type LegacyXlsConverter = (input: PassengerRosterWorkbookInput) => Promise<Buffer>;
|
||||
|
||||
export interface PassengerRosterWorkbookOptions {
|
||||
convertLegacyXls?: LegacyXlsConverter;
|
||||
maxInputBytes?: number;
|
||||
maxUncompressedBytes?: number;
|
||||
maxArchiveEntries?: number;
|
||||
maxDataRows?: number;
|
||||
}
|
||||
|
||||
export interface NormalizedPassengerRosterWorkbook {
|
||||
canonicalTsv: string;
|
||||
rowCount: number;
|
||||
sourceFormat: 'xls' | 'xlsx';
|
||||
worksheetName: string;
|
||||
headerRow: number;
|
||||
}
|
||||
|
||||
type ScalarCellValue = string | number | boolean | Date | null;
|
||||
|
||||
function fail(
|
||||
code: PassengerRosterWorkbookErrorCode,
|
||||
location: { row?: number; column?: number } = {}
|
||||
): never {
|
||||
throw new PassengerRosterWorkbookError(code, location);
|
||||
}
|
||||
|
||||
function startsWithMagic(content: Buffer, magic: Buffer): boolean {
|
||||
return content.byteLength >= magic.byteLength
|
||||
&& content.subarray(0, magic.byteLength).equals(magic);
|
||||
}
|
||||
|
||||
function normalizedMimeType(contentType: string): string {
|
||||
return contentType.toLowerCase().split(';', 1)[0].trim();
|
||||
}
|
||||
|
||||
function sourceFormat(input: PassengerRosterWorkbookInput): 'xls' | 'xlsx' {
|
||||
const extension = extname(input.fileName).toLowerCase();
|
||||
const mimeType = normalizedMimeType(input.contentType);
|
||||
const xlsxMagic = startsWithMagic(input.content, XLSX_MAGIC);
|
||||
const xlsMagic = startsWithMagic(input.content, XLS_MAGIC);
|
||||
const genericMime = mimeType === ''
|
||||
|| mimeType === 'application/octet-stream'
|
||||
|| mimeType === 'application/zip';
|
||||
const xlsxMime = mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet';
|
||||
const xlsMime = [
|
||||
'application/vnd.ms-excel',
|
||||
'application/msexcel',
|
||||
'application/x-msexcel',
|
||||
'application/xls',
|
||||
'application/vnd.ms-office'
|
||||
].includes(mimeType);
|
||||
|
||||
if (extension !== '.xls' && extension !== '.xlsx') {
|
||||
fail('roster_workbook_unsupported_format');
|
||||
}
|
||||
if (extension === '.xlsx') {
|
||||
if (!xlsxMagic || xlsMagic || (!genericMime && !xlsxMime)) {
|
||||
fail('roster_workbook_format_mismatch');
|
||||
}
|
||||
return 'xlsx';
|
||||
}
|
||||
if (!xlsMagic || xlsxMagic || (!genericMime && !xlsMime)) {
|
||||
fail('roster_workbook_format_mismatch');
|
||||
}
|
||||
return 'xls';
|
||||
}
|
||||
|
||||
function legacyWorkbookHasMacroMarkers(content: Buffer): boolean {
|
||||
const markers = ['_VBA_PROJECT_CUR', 'PROJECTwm', 'VBA/dir', 'VBA\\dir', 'Macros'];
|
||||
return markers.some((marker) => (
|
||||
content.indexOf(Buffer.from(marker, 'latin1')) >= 0
|
||||
|| content.indexOf(Buffer.from(marker, 'utf16le')) >= 0
|
||||
));
|
||||
}
|
||||
|
||||
const UNSAFE_ARCHIVE_PATH = /(?:^|\/)(?:vbaProject\.bin|macrosheets\/|activex\/|embeddings\/|externalLinks\/|connections\.xml$|queryTables\/|webextensions\/|model\/)/i;
|
||||
const RELATIONSHIP_OR_CONTENT_TYPE_PATH = /(?:\.rels|\[Content_Types\]\.xml)$/i;
|
||||
const EXTERNAL_RELATIONSHIP_PATTERN = /\bTargetMode\s*=\s*["']External["']/i;
|
||||
const MACRO_CONTENT_TYPE_PATTERN = /(?:macroEnabled|vbaProject|activeX|oleObject)/i;
|
||||
|
||||
async function readArchiveEntryText(zipFile: ZipFile, entry: Entry): Promise<string> {
|
||||
const stream = await zipFile.openReadStreamPromise(entry);
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks).toString('utf8');
|
||||
}
|
||||
|
||||
async function inspectXlsxArchive(
|
||||
content: Buffer,
|
||||
options: Required<Pick<PassengerRosterWorkbookOptions, 'maxUncompressedBytes' | 'maxArchiveEntries'>>
|
||||
): Promise<void> {
|
||||
let zipFile: ZipFile;
|
||||
try {
|
||||
zipFile = await fromBufferPromise(content, {
|
||||
lazyEntries: true,
|
||||
decodeStrings: true,
|
||||
validateEntrySizes: true,
|
||||
strictFileNames: true
|
||||
});
|
||||
} catch {
|
||||
fail('roster_workbook_invalid_xlsx');
|
||||
}
|
||||
|
||||
let entryCount = 0;
|
||||
let uncompressedBytes = 0;
|
||||
try {
|
||||
for await (const entry of zipFile.eachEntry()) {
|
||||
entryCount += 1;
|
||||
uncompressedBytes += entry.uncompressedSize;
|
||||
if (
|
||||
entryCount > options.maxArchiveEntries
|
||||
|| entry.uncompressedSize > options.maxUncompressedBytes
|
||||
|| uncompressedBytes > options.maxUncompressedBytes
|
||||
) {
|
||||
fail('roster_workbook_archive_limit');
|
||||
}
|
||||
if (entry.isEncrypted() || !entry.canDecodeFileData()) {
|
||||
fail('roster_workbook_unsafe_content');
|
||||
}
|
||||
const normalizedPath = entry.fileName.replace(/\\/g, '/');
|
||||
if (UNSAFE_ARCHIVE_PATH.test(normalizedPath)) {
|
||||
fail('roster_workbook_unsafe_content');
|
||||
}
|
||||
if (RELATIONSHIP_OR_CONTENT_TYPE_PATH.test(normalizedPath) && entry.uncompressedSize > 0) {
|
||||
const source = await readArchiveEntryText(zipFile, entry);
|
||||
if (
|
||||
EXTERNAL_RELATIONSHIP_PATTERN.test(source)
|
||||
|| (normalizedPath.endsWith('[Content_Types].xml') && MACRO_CONTENT_TYPE_PATTERN.test(source))
|
||||
) {
|
||||
fail('roster_workbook_unsafe_content');
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof PassengerRosterWorkbookError) throw error;
|
||||
fail('roster_workbook_invalid_xlsx');
|
||||
} finally {
|
||||
zipFile.close();
|
||||
}
|
||||
}
|
||||
|
||||
function formulaText(cell: ExcelJS.Cell): string | undefined {
|
||||
if (cell.type !== ExcelJS.ValueType.Formula) return undefined;
|
||||
const formula = cell.formula;
|
||||
return typeof formula === 'string' ? formula : undefined;
|
||||
}
|
||||
|
||||
function containsExternalFormulaReference(formula: string): boolean {
|
||||
return /(?:\[[^\]]+\]|https?:|ftp:|file:|\\\\|WEBSERVICE\s*\(|HYPERLINK\s*\(|DDE\s*\()/i.test(formula);
|
||||
}
|
||||
|
||||
function normalizedFormula(formula: string): string {
|
||||
return formula
|
||||
.replace(/^=/, '')
|
||||
.replace(/\$/g, '')
|
||||
.replace(/;/g, ',')
|
||||
.replace(/\s+/g, '')
|
||||
.toUpperCase();
|
||||
}
|
||||
|
||||
function cellRow(cell: ExcelJS.Cell): number {
|
||||
return cell.fullAddress.row;
|
||||
}
|
||||
|
||||
function cellColumn(cell: ExcelJS.Cell): number {
|
||||
return cell.fullAddress.col;
|
||||
}
|
||||
|
||||
function formulaAllowed(cell: ExcelJS.Cell, headerRow: number): boolean {
|
||||
const row = cellRow(cell);
|
||||
const column = cellColumn(cell);
|
||||
if (row <= headerRow) return false;
|
||||
const formula = normalizedFormula(cell.formula);
|
||||
if (column === 7) {
|
||||
return formula === `DATEDIF(F${row},TODAY(),"Y")`;
|
||||
}
|
||||
if (column === 12) {
|
||||
return formula === `EDATE(K${row},10*12)-1`;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function scalarValue(cell: ExcelJS.Cell, allowFormulaResult = false): ScalarCellValue {
|
||||
if (cell.type === ExcelJS.ValueType.Formula) {
|
||||
if (!allowFormulaResult) fail('roster_workbook_formula_not_allowed', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
const result = cell.result;
|
||||
if (result === undefined || result === null) {
|
||||
fail('roster_workbook_formula_result_missing', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
if (typeof result === 'object' && !(result instanceof Date)) {
|
||||
fail('roster_workbook_invalid_cell', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const value = cell.value;
|
||||
if (value === undefined || value === null) return null;
|
||||
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean' || value instanceof Date) {
|
||||
return value;
|
||||
}
|
||||
if ('hyperlink' in value) {
|
||||
fail('roster_workbook_unsafe_content', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
if ('richText' in value) {
|
||||
return value.richText.map((part) => part.text).join('');
|
||||
}
|
||||
fail('roster_workbook_invalid_cell', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
|
||||
function normalizedText(value: ScalarCellValue): string {
|
||||
if (value === null) return '';
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
return String(value)
|
||||
.replace(/[\t\r\n\u3000]+/g, ' ')
|
||||
.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function headerText(cell: ExcelJS.Cell): string {
|
||||
if (cell.type === ExcelJS.ValueType.Formula) return '';
|
||||
return normalizedText(scalarValue(cell));
|
||||
}
|
||||
|
||||
function cellHasData(cell: ExcelJS.Cell): boolean {
|
||||
if (cell.type === ExcelJS.ValueType.Formula) return true;
|
||||
return normalizedText(scalarValue(cell)) !== '';
|
||||
}
|
||||
|
||||
function rowHasData(row: ExcelJS.Row, maxColumn = SOURCE_COLUMN_COUNT): boolean {
|
||||
for (let column = 1; column <= maxColumn; column += 1) {
|
||||
if (cellHasData(row.getCell(column))) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function validateFormulasAndHiddenData(sheet: ExcelJS.Worksheet, headerRow: number): void {
|
||||
sheet.eachRow({ includeEmpty: false }, (row, rowNumber) => {
|
||||
let hasData = false;
|
||||
row.eachCell({ includeEmpty: false }, (cell, columnNumber) => {
|
||||
if (cell.type === ExcelJS.ValueType.Merge && cell.master.address !== cell.address) return;
|
||||
const formula = formulaText(cell);
|
||||
if (formula) {
|
||||
if (containsExternalFormulaReference(formula)) {
|
||||
fail('roster_workbook_unsafe_content', { row: rowNumber, column: columnNumber });
|
||||
}
|
||||
if (!formulaAllowed(cell, headerRow)) {
|
||||
fail('roster_workbook_formula_not_allowed', { row: rowNumber, column: columnNumber });
|
||||
}
|
||||
}
|
||||
if (cell.type === ExcelJS.ValueType.Hyperlink || cell.isHyperlink) {
|
||||
fail('roster_workbook_unsafe_content', { row: rowNumber, column: columnNumber });
|
||||
}
|
||||
if (cellHasData(cell)) hasData = true;
|
||||
if (columnNumber > SOURCE_COLUMN_COUNT && rowNumber >= headerRow) {
|
||||
fail('roster_workbook_unexpected_column', { row: rowNumber, column: columnNumber });
|
||||
}
|
||||
});
|
||||
if (row.hidden && hasData) fail('roster_workbook_hidden_data', { row: rowNumber });
|
||||
});
|
||||
|
||||
for (let columnNumber = 1; columnNumber <= sheet.columnCount; columnNumber += 1) {
|
||||
const column = sheet.getColumn(columnNumber);
|
||||
if (!column.hidden) continue;
|
||||
let hasData = false;
|
||||
column.eachCell({ includeEmpty: false }, (cell) => {
|
||||
if (cellHasData(cell)) hasData = true;
|
||||
});
|
||||
if (hasData) fail('roster_workbook_hidden_data', { column: columnNumber });
|
||||
}
|
||||
}
|
||||
|
||||
function findHeaderRow(sheet: ExcelJS.Worksheet): number {
|
||||
const candidates: number[] = [];
|
||||
const lastCandidate = Math.min(sheet.rowCount, MAX_HEADER_ROW);
|
||||
for (let rowNumber = 1; rowNumber <= lastCandidate; rowNumber += 1) {
|
||||
const row = sheet.getRow(rowNumber);
|
||||
const values = SOURCE_HEADERS.map((_header, index) => headerText(row.getCell(index + 1)));
|
||||
if (values.every((value, index) => value === SOURCE_HEADERS[index])) {
|
||||
candidates.push(rowNumber);
|
||||
}
|
||||
}
|
||||
if (candidates.length === 0) fail('roster_workbook_header_not_found');
|
||||
if (candidates.length !== 1) fail('roster_workbook_ambiguous_header');
|
||||
if (candidates[0] !== EXPECTED_HEADER_ROW) {
|
||||
fail('roster_workbook_header_row_invalid', { row: candidates[0] });
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
function excelSerialToDate(value: number, date1904: boolean): Date | undefined {
|
||||
if (!Number.isFinite(value) || value < 0) return undefined;
|
||||
const millisecondsPerDay = 86_400_000;
|
||||
const epoch = date1904
|
||||
? Date.UTC(1904, 0, 1)
|
||||
: Date.UTC(1899, 11, 30);
|
||||
const result = new Date(epoch + Math.floor(value) * millisecondsPerDay);
|
||||
return Number.isNaN(result.getTime()) ? undefined : result;
|
||||
}
|
||||
|
||||
function canonicalDate(cell: ExcelJS.Cell, date1904: boolean, formulaResult = false): string {
|
||||
const value = scalarValue(cell, formulaResult);
|
||||
if (value === null || value === '') return '';
|
||||
let date: Date | undefined;
|
||||
if (value instanceof Date) {
|
||||
date = value;
|
||||
} else if (typeof value === 'number') {
|
||||
date = excelSerialToDate(value, date1904);
|
||||
} else if (typeof value === 'string') {
|
||||
const match = value.trim().match(/^(\d{4})[-/.\u5e74](\d{1,2})[-/.\u6708](\d{1,2})(?:\u65e5)?$/);
|
||||
if (match) {
|
||||
date = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3])));
|
||||
if (
|
||||
date.getUTCFullYear() !== Number(match[1])
|
||||
|| date.getUTCMonth() !== Number(match[2]) - 1
|
||||
|| date.getUTCDate() !== Number(match[3])
|
||||
) {
|
||||
date = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!date || Number.isNaN(date.getTime())) {
|
||||
fail('roster_workbook_invalid_date', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
const year = date.getUTCFullYear();
|
||||
if (year < 1900 || year > 2200) {
|
||||
fail('roster_workbook_invalid_date', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
return `${String(year).padStart(4, '0')}-${String(date.getUTCMonth() + 1).padStart(2, '0')}-${String(date.getUTCDate()).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
function canonicalIdentifier(cell: ExcelJS.Cell): string {
|
||||
const value = scalarValue(cell);
|
||||
if (value === null || value === '') return '';
|
||||
if (typeof value === 'number') {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
fail('roster_workbook_invalid_identifier', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
if (typeof value !== 'string') {
|
||||
fail('roster_workbook_invalid_identifier', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
return normalizedText(value);
|
||||
}
|
||||
|
||||
function canonicalSequence(cell: ExcelJS.Cell): number {
|
||||
const value = scalarValue(cell);
|
||||
let sequence: number;
|
||||
if (typeof value === 'number') {
|
||||
sequence = value;
|
||||
} else if (typeof value === 'string' && /^[1-9]\d*$/.test(value.trim())) {
|
||||
sequence = Number(value.trim());
|
||||
} else {
|
||||
fail('roster_workbook_invalid_sequence', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
if (!Number.isSafeInteger(sequence) || sequence <= 0) {
|
||||
fail('roster_workbook_invalid_sequence', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
if (sequence > MAX_PASSENGER_SEQUENCE) {
|
||||
fail('roster_workbook_sequence_limit', { row: cellRow(cell), column: cellColumn(cell) });
|
||||
}
|
||||
return sequence;
|
||||
}
|
||||
|
||||
function canonicalTextCell(cell: ExcelJS.Cell): string {
|
||||
return normalizedText(scalarValue(cell));
|
||||
}
|
||||
|
||||
function dataRows(sheet: ExcelJS.Worksheet, headerRow: number, maxDataRows: number): ExcelJS.Row[] {
|
||||
const rows: ExcelJS.Row[] = [];
|
||||
let gapAfterData = false;
|
||||
for (let rowNumber = headerRow + 1; rowNumber <= sheet.rowCount; rowNumber += 1) {
|
||||
const row = sheet.getRow(rowNumber);
|
||||
const hasData = rowHasData(row);
|
||||
if (!hasData) {
|
||||
if (rows.length > 0) gapAfterData = true;
|
||||
continue;
|
||||
}
|
||||
if (gapAfterData) fail('roster_workbook_non_contiguous_rows', { row: rowNumber });
|
||||
rows.push(row);
|
||||
if (rows.length > maxDataRows) fail('roster_workbook_row_limit');
|
||||
}
|
||||
if (rows.length === 0) fail('roster_workbook_no_data');
|
||||
return rows;
|
||||
}
|
||||
|
||||
function canonicalRow(row: ExcelJS.Row, date1904: boolean, seenSequences: Set<number>): string[] {
|
||||
const sequence = canonicalSequence(row.getCell(1));
|
||||
if (seenSequences.has(sequence)) {
|
||||
fail('roster_workbook_duplicate_sequence', { row: row.number, column: 1 });
|
||||
}
|
||||
seenSequences.add(sequence);
|
||||
|
||||
if (canonicalIdentifier(row.getCell(5)) !== '') {
|
||||
fail('roster_workbook_identity_card_not_supported', { row: row.number, column: 5 });
|
||||
}
|
||||
|
||||
const englishName = canonicalTextCell(row.getCell(3)).replace(/，/g, ',');
|
||||
return [
|
||||
String(sequence),
|
||||
canonicalTextCell(row.getCell(2)),
|
||||
englishName,
|
||||
canonicalTextCell(row.getCell(4)),
|
||||
canonicalDate(row.getCell(6), date1904),
|
||||
canonicalTextCell(row.getCell(8)),
|
||||
'护照',
|
||||
canonicalIdentifier(row.getCell(9)),
|
||||
canonicalTextCell(row.getCell(10)),
|
||||
canonicalDate(row.getCell(11), date1904),
|
||||
canonicalDate(row.getCell(12), date1904, row.getCell(12).type === ExcelJS.ValueType.Formula),
|
||||
canonicalIdentifier(row.getCell(13)),
|
||||
canonicalTextCell(row.getCell(14))
|
||||
];
|
||||
}
|
||||
|
||||
export async function normalizePassengerRosterWorkbook(
|
||||
input: PassengerRosterWorkbookInput,
|
||||
options: PassengerRosterWorkbookOptions = {}
|
||||
): Promise<NormalizedPassengerRosterWorkbook> {
|
||||
const maxInputBytes = options.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES;
|
||||
const maxUncompressedBytes = options.maxUncompressedBytes ?? DEFAULT_MAX_UNCOMPRESSED_BYTES;
|
||||
const maxArchiveEntries = options.maxArchiveEntries ?? DEFAULT_MAX_ARCHIVE_ENTRIES;
|
||||
const requestedMaxDataRows = options.maxDataRows ?? DEFAULT_MAX_DATA_ROWS;
|
||||
const maxDataRows = Math.min(requestedMaxDataRows, ABSOLUTE_MAX_DATA_ROWS);
|
||||
|
||||
if (!Number.isSafeInteger(maxInputBytes) || maxInputBytes <= 0) fail('roster_workbook_too_large');
|
||||
if (!Number.isSafeInteger(maxUncompressedBytes) || maxUncompressedBytes <= 0) fail('roster_workbook_archive_limit');
|
||||
if (!Number.isSafeInteger(maxArchiveEntries) || maxArchiveEntries <= 0) fail('roster_workbook_archive_limit');
|
||||
if (!Number.isSafeInteger(requestedMaxDataRows) || requestedMaxDataRows <= 0) fail('roster_workbook_row_limit');
|
||||
if (input.content.byteLength === 0) fail('roster_workbook_empty');
|
||||
if (input.content.byteLength > maxInputBytes) fail('roster_workbook_too_large');
|
||||
|
||||
const format = sourceFormat(input);
|
||||
let xlsxContent = input.content;
|
||||
if (format === 'xls') {
|
||||
if (legacyWorkbookHasMacroMarkers(input.content)) fail('roster_workbook_unsafe_content');
|
||||
if (!options.convertLegacyXls) fail('roster_workbook_converter_unavailable');
|
||||
try {
|
||||
xlsxContent = await options.convertLegacyXls(input);
|
||||
} catch {
|
||||
fail('roster_workbook_conversion_failed');
|
||||
}
|
||||
if (
|
||||
!Buffer.isBuffer(xlsxContent)
|
||||
|| xlsxContent.byteLength === 0
|
||||
|| xlsxContent.byteLength > maxInputBytes
|
||||
|| !startsWithMagic(xlsxContent, XLSX_MAGIC)
|
||||
) {
|
||||
fail('roster_workbook_conversion_failed');
|
||||
}
|
||||
}
|
||||
|
||||
await inspectXlsxArchive(xlsxContent, { maxUncompressedBytes, maxArchiveEntries });
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
try {
|
||||
const excelJsBuffer = xlsxContent as unknown as Parameters<typeof workbook.xlsx.load>[0];
|
||||
await workbook.xlsx.load(excelJsBuffer);
|
||||
} catch {
|
||||
fail('roster_workbook_invalid_xlsx');
|
||||
}
|
||||
if (workbook.worksheets.length !== 1) fail('roster_workbook_ambiguous_sheet');
|
||||
const sheet = workbook.worksheets[0];
|
||||
if (sheet.state !== 'visible') fail('roster_workbook_hidden_data');
|
||||
const headerRow = findHeaderRow(sheet);
|
||||
validateFormulasAndHiddenData(sheet, headerRow);
|
||||
const rows = dataRows(sheet, headerRow, maxDataRows);
|
||||
const date1904 = workbook.properties.date1904 === true;
|
||||
const seenSequences = new Set<number>();
|
||||
const canonicalRows = rows.map((row) => canonicalRow(row, date1904, seenSequences));
|
||||
return {
|
||||
canonicalTsv: [CANONICAL_PASSENGER_HEADERS, ...canonicalRows]
|
||||
.map((row) => row.join('\t'))
|
||||
.join('\n'),
|
||||
rowCount: canonicalRows.length,
|
||||
sourceFormat: format,
|
||||
worksheetName: sheet.name,
|
||||
headerRow
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user