shiyuyun/wyndham-ARR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: move report artifacts to OSS storage

Browse Source
This commit is contained in:
Wyndham ARR
2026-08-04 12:37:26 +08:00
parent 727643f1f2
commit a2b86cdf10
45 changed files with 1288 additions and 1417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
company_reports/README.md
View File

@@ -28,7 +28,7 @@ python3 -m company_reports generate \
--as-of 2026-07-10
```
The workbook builder uses Python/openpyxl. No Node.js or `@oai/artifact-tool` package is required for company reports.
The workbook builder uses Python/openpyxl. The XLSX and `result.json` are uploaded to the existing private OSS; `.web-jobs` remains a local persistent queue-state directory. Use the existing `ARR_OBJECT_PREFIX`, `ARR_OSS_REGION`, `ARR_OSS_BUCKET`, optional `ARR_OSS_ENDPOINT`, and OSS credential settings; no Node.js or private package setting is required.
The DSN can fall back to `ARR_DATABASE_URL`; never place a real DSN in source, prompts, output JSON or browser code.
Scheduling convention:

43
company_reports/cli.py
View File

@@ -11,7 +11,8 @@ from pathlib import Path
from typing import Optional, Sequence, Tuple
from company_reports.contracts import COMPANY_NAMES, ErrorCode, RESULT_SCHEMA_VERSION
from company_reports.publishing import ArtifactToolBuilder, AtomicReportPublisher
from arr_web.processing_runtime import compose_object_store
from company_reports.publishing import OpenpyxlWorkbookBuilder, AtomicReportPublisher
from company_reports.repository import DatabaseConfig, PostgresReportRepository, RepositoryError
from company_reports.service import CompanyReportService, RunRequest, write_batch_result
@@ -69,14 +70,6 @@ def _parser() -> SafeArgumentParser:
"--output-root",
help="controlled output root inside the project",
)
generate.add_argument(
"--node-binary",
help="deprecated compatibility option; ignored by the openpyxl builder",
)
generate.add_argument(
"--artifact-tool-module",
help="deprecated compatibility option; ignored by the openpyxl builder",
)
return parser
@@ -110,23 +103,21 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
output_root = output_root.resolve()
config = DatabaseConfig.from_environment()
repository = PostgresReportRepository(config)
builder = ArtifactToolBuilder(
PROJECT_ROOT / "company_reports" / "xlsx" / "build_workbook.mjs",
node_binary=args.node_binary,
artifact_tool_module=(
Path(args.artifact_tool_module).expanduser()
if args.artifact_tool_module
else None
),
)
publisher = AtomicReportPublisher(PROJECT_ROOT, output_root)
service = CompanyReportService(
repository,
builder,
publisher,
output_root / ".staging",
)
result = service.run(request)
storage_runtime = compose_object_store()
try:
service = CompanyReportService(
repository,
OpenpyxlWorkbookBuilder(),
AtomicReportPublisher(
PROJECT_ROOT,
output_root,
object_store=storage_runtime.object_store,
),
output_root / ".staging",
)
result = service.run(request)
finally:
storage_runtime.close()
result_path = (
output_root
/ f"{year:04d}"

137
company_reports/publishing.py
View File

@@ -17,6 +17,7 @@ from openpyxl import Workbook, load_workbook
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
from openpyxl.worksheet.worksheet import Worksheet
from arr_storage.store import ManagedObjectStore
from company_reports.contracts import CompanyReport, ErrorCode, RESULT_SCHEMA_VERSION
from company_reports.repository import (
FileMetadata,
@@ -100,16 +101,11 @@ def _private_json(path: Path, payload: Mapping[str, Any]) -> None:
raise
class ArtifactToolBuilder:
def __init__(
self,
builder_script: Path,
node_binary: Optional[str] = None,
artifact_tool_module: Optional[Path] = None,
timeout_seconds: int = 120,
) -> None:
_ = (node_binary, artifact_tool_module, timeout_seconds)
self._builder_script = builder_script.resolve()
class OpenpyxlWorkbookBuilder:
def __init__(self, *_legacy_args: Any, **_legacy_options: Any) -> None:
# Retain import/call compatibility for older wrappers; no external
# workbook runtime is consulted.
_ = (_legacy_args, _legacy_options)
@staticmethod
def _safe_text(value: Any) -> str:
@@ -324,11 +320,19 @@ class ArtifactToolBuilder:
summary=summary,
)
class AtomicReportPublisher:
def __init__(self, project_root: Path, output_root: Path) -> None:
def __init__(
self,
project_root: Path,
output_root: Path,
*,
object_store: Optional[ManagedObjectStore] = None,
bucket_alias: str = "arr-private",
) -> None:
self._project_root = project_root.resolve()
self._output_root = output_root.resolve()
self._object_store = object_store
self._bucket_alias = bucket_alias
try:
self._output_root.relative_to(self._project_root)
except ValueError:
@@ -441,13 +445,41 @@ class AtomicReportPublisher:
if archive_sha256 != existing_sha256:
raise ValueError("archive and result JSON hashes differ")
artifact_storage_key = self._storage_key(archive_path)
storage_provider = str(artifact_payload.get("storage_provider") or "local")
bucket_alias = str(
artifact_payload.get("bucket_alias") or "arr-project-root"
)
if storage_provider in {"oss", "s3"}:
if self._object_store is None:
raise ValueError("OSS artifact reader is unavailable")
artifact_storage_key = str(artifact_payload.get("storage_key") or "")
stored = self._object_store.inspect_committed(
artifact_storage_key,
report.filename,
)
if (
stored.role != "company_report"
or stored.sha256 != existing_sha256
or stored.byte_size != archive_path.stat().st_size
or stored.mime_type != XLSX_MIME
):
raise ValueError("OSS archive and local cache differ")
artifact_byte_size = stored.byte_size
artifact_mime_type = stored.mime_type
elif storage_provider in {"local", "local_fixture"}:
artifact_storage_key = self._storage_key(archive_path)
artifact_byte_size = archive_path.stat().st_size
artifact_mime_type = XLSX_MIME
else:
raise ValueError("artifact storage provider is invalid")
expected_payload = self._success_result(
report,
reservation,
artifact_storage_key,
existing_sha256,
semantic_sha256,
storage_provider,
bucket_alias,
)
if result_payload != expected_payload:
raise ValueError("result JSON identity differs")
@@ -457,16 +489,38 @@ class AtomicReportPublisher:
original_filename=report.filename,
storage_key=artifact_storage_key,
sha256=existing_sha256,
byte_size=archive_path.stat().st_size,
mime_type=XLSX_MIME,
byte_size=artifact_byte_size,
mime_type=artifact_mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
result_sha256 = sha256_file(result_path)
result_byte_size = result_path.stat().st_size
result_storage_key = self._storage_key(result_path)
result_mime_type = "application/json"
if storage_provider in {"oss", "s3"}:
uploaded_result = self._object_store.upload_committed(
job_id=f"company-report-{reservation.report_version_id}",
attempt_no=1,
role="result_json",
source=result_path,
original_filename=result_path.name,
expected_sha256=result_sha256,
expected_byte_size=result_byte_size,
)
result_storage_key = uploaded_result.object_key
result_sha256 = uploaded_result.sha256
result_byte_size = uploaded_result.byte_size
result_mime_type = uploaded_result.mime_type
result_json = FileMetadata(
file_kind="result_json",
original_filename=result_path.name,
storage_key=self._storage_key(result_path),
sha256=sha256_file(result_path),
byte_size=result_path.stat().st_size,
mime_type="application/json",
storage_key=result_storage_key,
sha256=result_sha256,
byte_size=result_byte_size,
mime_type=result_mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
artifact.validate()
result_json.validate()
@@ -491,6 +545,8 @@ class AtomicReportPublisher:
artifact_storage_key: str,
artifact_sha256: str,
semantic_sha256: str,
storage_provider: str,
bucket_alias: str,
) -> Dict[str, Any]:
return {
"schema_version": RESULT_SCHEMA_VERSION,
@@ -506,6 +562,8 @@ class AtomicReportPublisher:
"storage_key": artifact_storage_key,
"sha256": artifact_sha256,
"semantic_sha256": semantic_sha256,
"storage_provider": storage_provider,
"bucket_alias": bucket_alias,
},
"row_count": report.row_count,
"period_row_counts": {
@@ -581,13 +639,30 @@ class AtomicReportPublisher:
archive_created = self._install_once(
built.path, archive_path, built.sha256
)
storage_provider = "local"
bucket_alias = "arr-project-root"
artifact_storage_key = self._storage_key(archive_path)
if self._object_store is not None:
uploaded_artifact = self._object_store.upload_committed(
job_id=f"company-report-{reservation.report_version_id}",
attempt_no=1,
role="company_report",
source=archive_path,
original_filename=report.filename,
expected_sha256=built.sha256,
expected_byte_size=built.byte_size,
)
storage_provider = "oss"
bucket_alias = self._bucket_alias
artifact_storage_key = uploaded_artifact.object_key
result_payload = self._success_result(
report,
reservation,
artifact_storage_key,
built.sha256,
str(built.summary.get("semantic_sha256", "")),
storage_provider,
bucket_alias,
)
if result_path.exists():
expected = json.dumps(
@@ -614,6 +689,8 @@ class AtomicReportPublisher:
sha256=built.sha256,
byte_size=built.byte_size,
mime_type=XLSX_MIME,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
result_json = FileMetadata(
file_kind="result_json",
@@ -622,7 +699,29 @@ class AtomicReportPublisher:
sha256=sha256_file(result_path),
byte_size=result_path.stat().st_size,
mime_type="application/json",
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
if self._object_store is not None:
uploaded_result = self._object_store.upload_committed(
job_id=f"company-report-{reservation.report_version_id}",
attempt_no=1,
role="result_json",
source=result_path,
original_filename=result_path.name,
expected_sha256=result_json.sha256,
expected_byte_size=result_json.byte_size,
)
result_json = FileMetadata(
file_kind="result_json",
original_filename=result_path.name,
storage_key=uploaded_result.object_key,
sha256=uploaded_result.sha256,
byte_size=uploaded_result.byte_size,
mime_type=uploaded_result.mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
repository.activate_report(reservation, artifact, result_json)
return PublicationOutcome(
current_path=current_path,

8
company_reports/repository.py
View File

@@ -112,6 +112,8 @@ class FileMetadata:
sha256: str
byte_size: int
mime_type: str
storage_provider: str = "local"
bucket_alias: str = "arr-project-root"
def validate(self) -> None:
storage = PurePosixPath(self.storage_key)
@@ -122,6 +124,12 @@ class FileMetadata:
or self.original_filename != PurePosixPath(self.original_filename).name
or not SHA256_RE.fullmatch(self.sha256)
or self.byte_size < 0
or not isinstance(self.storage_provider, str)
or self.storage_provider not in {"local", "local_fixture", "oss", "s3"}
or not isinstance(self.bucket_alias, str)
or not self.bucket_alias.strip()
or "/" in self.bucket_alias
or "\\" in self.bucket_alias
):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,

4
company_reports/service.py
View File

@@ -255,6 +255,10 @@ class CompanyReportService:
"filename": report.filename,
"storage_key": outcome.artifact.storage_key,
"sha256": outcome.artifact.sha256,
"byte_size": outcome.artifact.byte_size,
"mime_type": outcome.artifact.mime_type,
"storage_provider": outcome.artifact.storage_provider,
"bucket_alias": outcome.artifact.bucket_alias,
"semantic_sha256": str(
built.summary.get("semantic_sha256", "")
),

356
company_reports/xlsx/build_workbook.mjs
View File

@@ -1,356 +0,0 @@
import fs from "node:fs/promises";
import crypto from "node:crypto";
import path from "node:path";
import process from "node:process";
import { pathToFileURL } from "node:url";
let FileBlob;
let SpreadsheetFile;
let Workbook;
const EXPECTED_HEADERS = [
"ARRIVAL",
"DEPARTURE",
"NIGHTS",
"BLOCK_CODE",
"RES_COMMENT",
"Booking Room",
"Total Booking Price",
];
const DUPLICATE_FILL = "#FFF2CC";
const DUPLICATE_FONT = "#9C6500";
const REVIEW_FILL = "#FCE4D6";
const REVIEW_FONT = "#C65911";
const HEADER_FILL = "#FFFFFF";
const BODY_FONT = "#222222";
const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
async function loadArtifactTool() {
const configured = String(process.env.COMPANY_REPORT_ARTIFACT_TOOL_MODULE ?? "").trim();
const module = configured
? await import(pathToFileURL(path.resolve(configured)).href)
: await import("@oai/artifact-tool");
({ FileBlob, SpreadsheetFile, Workbook } = module);
if (!FileBlob || !SpreadsheetFile || !Workbook) fail("artifact-tool module is invalid");
}
function fail(message) {
throw new Error(message);
}
function safeText(value) {
const text = String(value ?? "");
return /^[=+\-@]/.test(text) ? `'${text}` : text;
}
function excelDate(value) {
if (!DATE_PATTERN.test(value)) fail("invalid date value");
const [year, month, day] = value.split("-").map(Number);
const parsed = new Date(Date.UTC(year, month - 1, day));
if (
parsed.getUTCFullYear() !== year
|| parsed.getUTCMonth() !== month - 1
|| parsed.getUTCDate() !== day
) {
fail("invalid date value");
}
return parsed;
}
function dateKey(value) {
if (value instanceof Date && !Number.isNaN(value.getTime())) {
return value.toISOString().slice(0, 10);
}
if (typeof value === "number" && Number.isFinite(value)) {
const epoch = Date.UTC(1899, 11, 30);
return new Date(epoch + Math.floor(value) * 86400000).toISOString().slice(0, 10);
}
if (typeof value === "string" && DATE_PATTERN.test(value.slice(0, 10))) {
return value.slice(0, 10);
}
return "";
}
function validatePayload(payload) {
if (!payload || payload.schema_version !== "1.0") fail("invalid payload schema");
if (!Array.isArray(payload.headers) || payload.headers.length !== EXPECTED_HEADERS.length) {
fail("invalid headers");
}
if (!payload.headers.every((header, index) => header === EXPECTED_HEADERS[index])) {
fail("invalid headers");
}
if (!Array.isArray(payload.periods) || payload.periods.length !== 3) {
fail("invalid period count");
}
const names = new Set();
for (const period of payload.periods) {
if (
typeof period.sheet_name !== "string"
|| period.sheet_name.length < 1
|| period.sheet_name.length > 31
|| names.has(period.sheet_name)
|| !Array.isArray(period.rows)
) {
fail("invalid worksheet contract");
}
names.add(period.sheet_name);
for (const row of period.rows) {
if (
!DATE_PATTERN.test(String(row.arrival ?? ""))
|| !DATE_PATTERN.test(String(row.departure ?? ""))
|| !Number.isInteger(row.nights)
|| row.nights < 0
|| typeof row.block_code !== "string"
|| typeof row.res_comment !== "string"
|| typeof row.booking_room !== "string"
|| typeof row.total_booking_price !== "string"
|| typeof row.duplicate_group !== "boolean"
|| typeof row.multi_price_review !== "boolean"
) {
fail("invalid output row contract");
}
}
}
}
function writeSheet(workbook, period, periodIndex) {
const sheet = workbook.worksheets.add(period.sheet_name);
const body = period.rows.map((row) => [
excelDate(row.arrival),
excelDate(row.departure),
row.nights,
safeText(row.block_code),
safeText(row.res_comment),
safeText(row.booking_room),
safeText(row.total_booking_price),
]);
const matrix = [EXPECTED_HEADERS, ...body];
const lastRow = matrix.length;
const used = sheet.getRange(`A1:G${lastRow}`);
used.values = matrix;
used.format.font = { name: "Arial", size: 10, color: BODY_FONT };
used.format.verticalAlignment = "center";
sheet.getRange("A1").format.columnWidth = 13;
sheet.getRange("B1").format.columnWidth = 13;
sheet.getRange("C1").format.columnWidth = 9;
sheet.getRange("D1").format.columnWidth = 20;
sheet.getRange("E1").format.columnWidth = 25;
sheet.getRange("F1").format.columnWidth = 34;
sheet.getRange("G1").format.columnWidth = 48;
const table = sheet.tables.add(
`A1:G${lastRow}`,
true,
`CompanyReportPeriod${periodIndex + 1}`,
);
table.style = "TableStyleLight1";
table.showHeaders = true;
table.showTotals = false;
table.showBandedColumns = false;
table.showFilterButton = true;
const header = sheet.getRange("A1:G1");
header.format.fill = HEADER_FILL;
header.format.font = { name: "Arial", size: 10, bold: true, color: "#000000" };
header.format.horizontalAlignment = "center";
header.format.verticalAlignment = "center";
header.format.rowHeight = 24;
header.format.borders = {
bottom: { style: "thin", color: "#7F7F7F" },
};
sheet.freezePanes.freezeRows(1);
if (body.length > 0) {
const data = sheet.getRange(`A2:G${lastRow}`);
data.format.rowHeight = 30;
sheet.getRange(`A2:B${lastRow}`).setNumberFormat("yyyy-mm-dd");
sheet.getRange(`A2:D${lastRow}`).format.horizontalAlignment = "center";
sheet.getRange(`E2:G${lastRow}`).format.horizontalAlignment = "left";
sheet.getRange(`D2:G${lastRow}`).format.wrapText = true;
period.rows.forEach((row, rowIndex) => {
const excelRow = rowIndex + 2;
if (row.duplicate_group) {
sheet.getRange(`A${excelRow}:G${excelRow}`).format.fill = DUPLICATE_FILL;
sheet.getRange(`E${excelRow}`).format.font = {
name: "Arial",
size: 10,
bold: true,
color: DUPLICATE_FONT,
};
}
if (row.multi_price_review) {
sheet.getRange(`G${excelRow}`).format.fill = REVIEW_FILL;
sheet.getRange(`G${excelRow}`).format.font = {
name: "Arial",
size: 10,
bold: true,
color: REVIEW_FONT,
};
}
});
data.format.autofitRows();
}
sheet.showGridLines = true;
return sheet;
}
async function validateWorkbook(workbook, payload, stage) {
const names = workbook.worksheets.items.map((sheet) => sheet.name);
const expectedNames = payload.periods.map((period) => period.sheet_name);
if (JSON.stringify(names) !== JSON.stringify(expectedNames)) {
fail(`${stage} worksheet names do not match`);
}
let formulaCount = 0;
for (let index = 0; index < payload.periods.length; index += 1) {
const period = payload.periods[index];
const sheet = workbook.worksheets.getItem(period.sheet_name);
const expectedRows = period.rows.length + 1;
const used = sheet.getUsedRange();
const values = used?.values ?? [];
if (values.length !== expectedRows || (values[0]?.length ?? 0) !== 7) {
fail(`${stage} worksheet dimensions do not match`);
}
if (!EXPECTED_HEADERS.every((header, column) => values[0][column] === header)) {
fail(`${stage} worksheet headers do not match`);
}
period.rows.forEach((row, rowIndex) => {
const actual = values[rowIndex + 1] ?? [];
const expected = [
row.arrival,
row.departure,
row.nights,
safeText(row.block_code),
safeText(row.res_comment),
safeText(row.booking_room),
safeText(row.total_booking_price),
];
if (
dateKey(actual[0]) !== expected[0]
|| dateKey(actual[1]) !== expected[1]
|| Number(actual[2]) !== expected[2]
|| String(actual[3] ?? "") !== expected[3]
|| String(actual[4] ?? "") !== expected[4]
|| String(actual[5] ?? "") !== expected[5]
|| String(actual[6] ?? "") !== expected[6]
) {
fail(`${stage} worksheet values do not match`);
}
});
if (sheet.tables.items.length !== 1 || !sheet.tables.items[0].showFilterButton) {
fail(`${stage} worksheet filter is missing`);
}
const formulaInspection = await workbook.inspect({
kind: "formula",
sheetId: period.sheet_name,
range: `A1:G${expectedRows}`,
maxChars: 4000,
options: { maxResults: 100 },
});
const text = String(formulaInspection.ndjson ?? "");
formulaCount += text
.split("\n")
.filter((line) => line.includes('"kind":"formula"')).length;
if (/#REF!|#DIV\/0!|#VALUE!|#NAME\?|#N\/A/.test(text)) {
fail(`${stage} workbook contains a formula error`);
}
}
if (formulaCount !== 0) fail(`${stage} workbook must contain no formulas`);
return formulaCount;
}
async function main() {
const [inputPath, outputPath, previewDir, summaryPath] = process.argv.slice(2);
if (!inputPath || !outputPath || !previewDir || !summaryPath) {
fail("usage: build_workbook.mjs INPUT_JSON OUTPUT_XLSX PREVIEW_DIR SUMMARY_JSON");
}
await loadArtifactTool();
const payload = JSON.parse(await fs.readFile(inputPath, "utf8"));
validatePayload(payload);
await fs.mkdir(path.dirname(outputPath), { recursive: true });
await fs.mkdir(previewDir, { recursive: true });
await fs.mkdir(path.dirname(summaryPath), { recursive: true });
const workbook = Workbook.create();
payload.periods.forEach((period, index) => writeSheet(workbook, period, index));
await validateWorkbook(workbook, payload, "pre-export");
const xlsx = await SpreadsheetFile.exportXlsx(workbook);
await xlsx.save(outputPath);
await fs.chmod(outputPath, 0o600);
const reopened = await SpreadsheetFile.importXlsx(await FileBlob.load(outputPath));
const formulaCount = await validateWorkbook(reopened, payload, "post-export");
const previews = [];
for (let index = 0; index < payload.periods.length; index += 1) {
const period = payload.periods[index];
const preview = await reopened.render({
sheetName: period.sheet_name,
autoCrop: "all",
scale: 1.5,
format: "png",
});
const previewPath = path.join(previewDir, `sheet-${index + 1}.png`);
await fs.writeFile(previewPath, new Uint8Array(await preview.arrayBuffer()), {
mode: 0o600,
});
previews.push(previewPath);
}
const stat = await fs.stat(outputPath);
const semanticSha256 = crypto
.createHash("sha256")
.update(JSON.stringify({ headers: payload.headers, periods: payload.periods }), "utf8")
.digest("hex");
const summary = {
status: "success",
schema_version: payload.schema_version,
company: payload.company,
filename: payload.filename,
sheet_names: payload.periods.map((period) => period.sheet_name),
row_counts: payload.periods.map((period) => period.rows.length),
formula_count: formulaCount,
semantic_sha256: semanticSha256,
preview_count: previews.length,
byte_size: stat.size,
};
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, {
encoding: "utf8",
mode: 0o600,
});
await fs.rm(`${outputPath}.inspect.ndjson`, { force: true });
process.stdout.write(`${JSON.stringify(summary)}\n`);
}
try {
await main();
} catch (_error) {
const outputPath = process.argv[3];
if (outputPath) {
await fs.rm(`${outputPath}.inspect.ndjson`, { force: true }).catch(() => undefined);
}
process.stderr.write(
`${JSON.stringify({
status: "failed",
code: "COMPANY_REPORT_OUTPUT_VALIDATION_FAILED",
})}\n`,
);
process.exitCode = 4;
}

5
company_reports/xlsx/package.json
View File

@@ -1,5 +0,0 @@
{
"name": "company-report-xlsx-builder",
"private": true,
"type": "module"
}