shiyuyun/wyndham-ARR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: move report artifacts to OSS storage

Browse Source
This commit is contained in:
Wyndham ARR
2026-08-04 12:37:26 +08:00
parent 727643f1f2
commit a2b86cdf10
45 changed files with 1288 additions and 1417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
arr_web/company_jobs.py
View File

@@ -18,6 +18,7 @@ from pathlib import Path, PurePosixPath
from typing import Any, Callable, Deque, Dict, List, Mapping, Optional, Protocol, Tuple
from zoneinfo import ZoneInfo
from arr_storage.store import ManagedObjectStore
from arr_web.contracts import PortalError, validate_month
from arr_web.downloads import ArtifactDescriptor, MAX_DOWNLOAD_BYTES
from company_reports.contracts import (
@@ -197,6 +198,7 @@ class PersistentCompanyReportCoordinator:
*,
jobs_root: Optional[Path] = None,
now: Optional[Callable[[], datetime]] = None,
object_store: Optional[ManagedObjectStore] = None,
) -> None:
self._project_root = project_root.resolve()
self._output_root = output_root.resolve()
@@ -213,6 +215,7 @@ class PersistentCompanyReportCoordinator:
self._jobs_root.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(self._jobs_root, 0o700)
self._executor = executor
self._object_store = object_store
self._now = now or (lambda: datetime.now(COMPANY_REPORT_TIME_ZONE))
self._condition = threading.Condition(threading.RLock())
self._pending: Deque[str] = deque()
@@ -612,6 +615,46 @@ class PersistentCompanyReportCoordinator:
or SHA256_RE.fullmatch(sha256) is None
):
raise ValueError("artifact metadata is invalid")
storage_provider = str(value.get("storage_provider") or "local")
bucket_alias = str(value.get("bucket_alias") or "arr-project-root")
byte_size_value = value.get("byte_size")
mime_type = str(value.get("mime_type") or XLSX_MIME)
if storage_provider in {"oss", "s3"}:
if self._object_store is None:
raise ValueError("OSS artifact reader is unavailable")
if (
isinstance(byte_size_value, bool)
or not isinstance(byte_size_value, int)
or byte_size_value <= 0
or mime_type != XLSX_MIME
):
raise ValueError("OSS artifact identity is invalid")
try:
stored = self._object_store.inspect_committed(
storage_key,
expected_filename,
)
except Exception:
raise ValueError("OSS artifact identity is unavailable") from None
if (
stored.role != "company_report"
or stored.sha256 != sha256
or stored.byte_size != byte_size_value
or stored.mime_type != mime_type
):
raise ValueError("OSS artifact identity does not match")
descriptor = ArtifactDescriptor(
file_kind="company_ten_day_xlsx",
original_filename=expected_filename,
storage_key=storage_key,
sha256=sha256,
byte_size=byte_size_value,
mime_type=mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
descriptor.validate()
return descriptor
storage = PurePosixPath(storage_key)
if storage.is_absolute() or not storage.parts or ".." in storage.parts or "." in storage.parts:
raise ValueError("artifact path is invalid")
@@ -637,6 +680,8 @@ class PersistentCompanyReportCoordinator:
sha256=sha256,
byte_size=metadata.st_size,
mime_type=XLSX_MIME,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
descriptor.validate()
return descriptor

33
arr_web/downloads.py
View File

@@ -29,6 +29,8 @@ class ArtifactDescriptor:
sha256: str
byte_size: int
mime_type: str
storage_provider: str = "local"
bucket_alias: str = "arr-project-root"
def validate(self) -> None:
path = PurePosixPath(self.storage_key)
@@ -48,6 +50,12 @@ class ArtifactDescriptor:
or any(character not in "0123456789abcdef" for character in self.sha256)
or not 0 < self.byte_size <= MAX_DOWNLOAD_BYTES
or not self.mime_type
or not isinstance(self.storage_provider, str)
or self.storage_provider not in {"local", "local_fixture", "oss", "s3"}
or not isinstance(self.bucket_alias, str)
or not self.bucket_alias
or "/" in self.bucket_alias
or "\\" in self.bucket_alias
):
raise PortalError("DOWNLOAD_REFERENCE_INVALID", "文件身份无效", 500)
@@ -63,14 +71,21 @@ class UnavailableArtifactReader:
class ManagedObjectArtifactReader:
"""Read a committed daily XLSX from ARR's immutable object store."""
"""Read a committed report artifact from ARR's immutable object store."""
_EXPECTED_ROLES = {
"daily_xlsx": "daily_report",
"monthly_xlsx": "monthly_report",
"company_ten_day_xlsx": "company_report",
}
def __init__(self, object_store: ManagedObjectStore) -> None:
self._object_store = object_store
def read(self, descriptor: ArtifactDescriptor) -> bytes:
descriptor.validate()
if descriptor.file_kind != "daily_xlsx":
expected_role = self._EXPECTED_ROLES.get(descriptor.file_kind)
if expected_role is None:
raise PortalError("DOWNLOAD_REFERENCE_INVALID", "文件身份无效", 500)
try:
stored = self._object_store.inspect_committed(
@@ -78,7 +93,7 @@ class ManagedObjectArtifactReader:
descriptor.original_filename,
)
if (
stored.role != "daily_report"
stored.role != expected_role
or stored.sha256 != descriptor.sha256
or stored.byte_size != descriptor.byte_size
or stored.mime_type != descriptor.mime_type
@@ -87,7 +102,7 @@ class ManagedObjectArtifactReader:
"DOWNLOAD_IDENTITY_MISMATCH", "文件完整性校验失败", 503
)
with tempfile.TemporaryDirectory(prefix="arr-download-") as temporary:
destination = Path(temporary) / "daily.xlsx"
destination = Path(temporary) / "artifact.bin"
self._object_store.materialize(
descriptor.storage_key,
destination,
@@ -111,16 +126,18 @@ class ManagedObjectArtifactReader:
class RoutedArtifactReader:
"""Route OSS daily artifacts and controlled local report artifacts safely."""
"""Route managed OSS artifacts and legacy controlled-local artifacts safely."""
def __init__(
self,
*,
daily_reader: Optional[ArtifactReader],
local_reader: ArtifactReader,
oss_reader: Optional[ArtifactReader] = None,
) -> None:
self._daily_reader = daily_reader
self._local_reader = local_reader
self._oss_reader = oss_reader
def read(self, descriptor: ArtifactDescriptor) -> bytes:
descriptor.validate()
@@ -130,6 +147,12 @@ class RoutedArtifactReader:
"DOWNLOAD_UNAVAILABLE", "文件读取服务暂不可用", 503
)
return self._daily_reader.read(descriptor)
if descriptor.storage_provider in {"oss", "s3"}:
if self._oss_reader is None:
raise PortalError(
"DOWNLOAD_UNAVAILABLE", "文件读取服务暂不可用", 503
)
return self._oss_reader.read(descriptor)
return self._local_reader.read(descriptor)

40
arr_web/processing_runtime.py
View File

@@ -29,11 +29,18 @@ class ProcessingInputRuntime:
self.oss_client.close()
def compose_programmatic_processing(
*,
project_root: Path,
connect: Optional[Callable[[str], Any]] = None,
) -> ProcessingInputRuntime:
@dataclass
class ObjectStoreRuntime:
"""Shared OSS/object-store runtime for report publishers and downloads."""
oss_client: AliyunOssV2Client
object_store: ManagedObjectStore
def close(self) -> None:
self.oss_client.close()
def compose_object_store() -> ObjectStoreRuntime:
oss_client = AliyunOssV2Client(AliyunOssConfig.from_environment())
try:
oss_client.assert_immutable_writes_supported()
@@ -41,6 +48,24 @@ def compose_programmatic_processing(
CloudObjectBackend(oss_client),
ObjectKeyPolicy(os.environ.get("ARR_OBJECT_PREFIX", "arr")),
)
return ObjectStoreRuntime(oss_client, object_store)
except Exception:
try:
oss_client.close()
except Exception:
pass
raise
def compose_programmatic_processing(
*,
project_root: Path,
connect: Optional[Callable[[str], Any]] = None,
) -> ProcessingInputRuntime:
storage = compose_object_store()
try:
oss_client = storage.oss_client
object_store = storage.object_store
database_config = (
DatabaseConfig("controlled")
if connect is not None
@@ -64,10 +89,7 @@ def compose_programmatic_processing(
)
return ProcessingInputRuntime(coordinator, oss_client, object_store)
except Exception:
try:
oss_client.close()
except Exception:
pass
storage.close()
raise

25
arr_web/repository.py
View File

@@ -414,6 +414,8 @@ ORDER BY period_start DESC
DAILY_DOWNLOAD_SQL = """
SELECT
artifact.artifact_kind,
artifact.storage_provider,
artifact.bucket_alias,
artifact.original_filename,
artifact.object_key,
artifact.sha256,
@@ -432,6 +434,8 @@ WHERE run.run_key = %s
MONTHLY_DOWNLOAD_SQL = """
SELECT
artifact.artifact_kind,
artifact.storage_provider,
artifact.bucket_alias,
artifact.original_filename,
artifact.object_key,
artifact.sha256,
@@ -443,7 +447,6 @@ JOIN ingestion.artifacts AS artifact
WHERE run.id = %s
AND run.report_status IN ('active', 'superseded')
AND artifact.artifact_kind = 'monthly_xlsx'
AND artifact.storage_provider = 'local'
""".strip()
@@ -815,13 +818,23 @@ class PostgresPortalRepository:
if len(rows) != 1:
raise PortalDataError("DOWNLOAD_NOT_FOUND", "文件不存在或尚未生成")
row = rows[0]
if len(row) >= 8:
provider = str(row[1] or "local")
bucket_alias = str(row[2] or "arr-project-root")
filename_index, key_index, hash_index, size_index, mime_index = 3, 4, 5, 6, 7
else:
provider = "local"
bucket_alias = "arr-project-root"
filename_index, key_index, hash_index, size_index, mime_index = 1, 2, 3, 4, 5
descriptor = ArtifactDescriptor(
file_kind=str(row[0] or ""),
original_filename=str(row[1] or ""),
storage_key=str(row[2] or ""),
sha256=str(row[3] or "").lower(),
byte_size=int(row[4]),
mime_type=str(row[5] or "application/octet-stream"),
original_filename=str(row[filename_index] or ""),
storage_key=str(row[key_index] or ""),
sha256=str(row[hash_index] or "").lower(),
byte_size=int(row[size_index]),
mime_type=str(row[mime_index] or "application/octet-stream"),
storage_provider=provider,
bucket_alias=bucket_alias,
)
descriptor.validate()
return descriptor

151
arr_web/run.py
View File

@@ -21,16 +21,18 @@ from arr_web.downloads import (
)
from arr_web.repository import PostgresPortalRepository, UnavailablePortalRepository
from arr_web.processing_runtime import (
ObjectStoreRuntime,
ProcessingInputRuntime,
compose_object_store,
compose_programmatic_processing,
)
from arr_web.server import serve
from arr_web.services import ProgramMonthlyCoordinator
from monthly_reports.publishing import ArtifactToolBuilder, AtomicReportPublisher
from monthly_reports.publishing import OpenpyxlWorkbookBuilder, AtomicReportPublisher
from monthly_reports.repository import DatabaseConfig, PostgresReportRepository
from monthly_reports.service import MonthlyReportService
from company_reports.publishing import (
ArtifactToolBuilder as CompanyArtifactToolBuilder,
OpenpyxlWorkbookBuilder as CompanyOpenpyxlWorkbookBuilder,
AtomicReportPublisher as AtomicCompanyReportPublisher,
)
from company_reports.repository import (
@@ -81,8 +83,6 @@ def _parser() -> argparse.ArgumentParser:
action="store_true",
help="mark browser session cookies Secure for an HTTPS deployment",
)
parser.add_argument("--node-binary", type=Path)
parser.add_argument("--artifact-tool-module", type=Path)
return parser
@@ -108,67 +108,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
repository = UnavailablePortalRepository()
database_ready = False
monthly = None
monthly_ready = False
if args.enable_monthly_generation and database_ready:
try:
report_repository = PostgresReportRepository(
DatabaseConfig("controlled"),
connect=connect,
) if connect is not None else PostgresReportRepository(DatabaseConfig.from_environment())
builder = ArtifactToolBuilder(
PROJECT_ROOT / "monthly_reports" / "xlsx" / "build_workbook.mjs",
node_binary=str(args.node_binary) if args.node_binary else None,
artifact_tool_module=args.artifact_tool_module,
)
output_root = PROJECT_ROOT / "outputs" / "monthly_reports"
monthly = ProgramMonthlyCoordinator(
MonthlyReportService(
report_repository,
builder,
AtomicReportPublisher(PROJECT_ROOT, output_root),
output_root / ".staging",
)
)
monthly_ready = True
except Exception:
monthly = None
monthly_ready = False
company_reports = None
company_reports_ready = False
if args.enable_company_reports and database_ready:
try:
company_repository = (
PostgresCompanyReportRepository(
CompanyDatabaseConfig("controlled"),
connect=connect,
)
if connect is not None
else PostgresCompanyReportRepository(
CompanyDatabaseConfig.from_environment()
)
)
company_builder = CompanyArtifactToolBuilder(
PROJECT_ROOT / "company_reports" / "xlsx" / "build_workbook.mjs",
node_binary=str(args.node_binary) if args.node_binary else None,
artifact_tool_module=args.artifact_tool_module,
)
company_output_root = PROJECT_ROOT / "outputs" / "company_reports"
company_service = CompanyReportService(
company_repository,
company_builder,
AtomicCompanyReportPublisher(PROJECT_ROOT, company_output_root),
company_output_root / ".staging",
)
company_reports = PersistentCompanyReportCoordinator(
PROJECT_ROOT,
company_output_root,
ProgramCompanyReportExecutor(company_service),
)
company_reports_ready = True
except Exception:
company_reports = None
company_reports_ready = False
processing_input: Optional[ProcessingInputRuntime] = None
processing_ready = False
if args.enable_processing and database_ready:
@@ -181,6 +120,81 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
except Exception:
processing_input = None
processing_ready = False
report_storage_runtime: Optional[ObjectStoreRuntime] = None
report_object_store = (
processing_input.object_store if processing_input is not None else None
)
if report_object_store is None and database_ready and (
args.enable_monthly_generation or args.enable_company_reports
):
try:
report_storage_runtime = compose_object_store()
report_object_store = report_storage_runtime.object_store
except Exception:
report_storage_runtime = None
report_object_store = None
monthly = None
monthly_ready = False
if args.enable_monthly_generation and database_ready and report_object_store is not None:
try:
report_repository = PostgresReportRepository(
DatabaseConfig("controlled"),
connect=connect,
) if connect is not None else PostgresReportRepository(DatabaseConfig.from_environment())
output_root = PROJECT_ROOT / "outputs" / "monthly_reports"
monthly = ProgramMonthlyCoordinator(
MonthlyReportService(
report_repository,
OpenpyxlWorkbookBuilder(),
AtomicReportPublisher(
PROJECT_ROOT,
output_root,
object_store=report_object_store,
),
output_root / ".staging",
)
)
monthly_ready = True
except Exception:
monthly = None
monthly_ready = False
company_reports = None
company_reports_ready = False
if args.enable_company_reports and database_ready and report_object_store is not None:
try:
company_repository = (
PostgresCompanyReportRepository(
CompanyDatabaseConfig("controlled"),
connect=connect,
)
if connect is not None
else PostgresCompanyReportRepository(
CompanyDatabaseConfig.from_environment()
)
)
company_output_root = PROJECT_ROOT / "outputs" / "company_reports"
company_service = CompanyReportService(
company_repository,
CompanyOpenpyxlWorkbookBuilder(),
AtomicCompanyReportPublisher(
PROJECT_ROOT,
company_output_root,
object_store=report_object_store,
),
company_output_root / ".staging",
)
company_reports = PersistentCompanyReportCoordinator(
PROJECT_ROOT,
company_output_root,
ProgramCompanyReportExecutor(company_service),
object_store=report_object_store,
)
company_reports_ready = True
except Exception:
company_reports = None
company_reports_ready = False
booking_sources = None
company_source_upload_ready = False
if args.enable_company_reports and database_ready and processing_input is not None:
@@ -217,6 +231,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
else None
),
local_reader=ControlledProjectArtifactReader(PROJECT_ROOT),
oss_reader=(
ManagedObjectArtifactReader(report_object_store)
if report_object_store is not None
else None
),
)
if database_ready
else None
@@ -239,6 +258,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
company_reports.close()
if processing_input is not None:
processing_input.close()
if report_storage_runtime is not None:
report_storage_runtime.close()
return 0