180 lines
9.2 KiB
Python
180 lines
9.2 KiB
Python
"""Start an explicitly isolated, persistent XML replay portal on loopback only."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import signal
|
|
import tempfile
|
|
import threading
|
|
|
|
from arr_ingestion.postgres import DatabaseConfig, PostgresIngestionRepository
|
|
from arr_ingestion.service import IngestionService
|
|
from arr_ingestion.validation import DeliveryValidator
|
|
from arr_processing.local import LocalDailyProcessor
|
|
from arr_processing.policy import load_processor_policy
|
|
from arr_storage.filesystem import FilesystemObjectBackend
|
|
from arr_storage.store import ManagedObjectStore
|
|
from arr_web.app import PortalApplication, RuntimeHealth
|
|
from arr_web.arr_downloads import PersistentARRDownloads
|
|
from arr_web.auth import LoginCredentials
|
|
from arr_web.downloads import ManagedObjectArtifactReader
|
|
from arr_web.local_replay_database import ReplayDatabase
|
|
from arr_web.local_xml_replay import (PROJECT, LocalReplayPortal, NativeXMLReplayExecutor, NativeXMLSnapshot,
|
|
SourceDateDownloads, document)
|
|
from arr_web.programmatic import ProgrammaticUploadCoordinator
|
|
from arr_web.repository import PostgresPortalRepository
|
|
from arr_web.server import serve
|
|
from integrations.ohip.capture_job import atomic_json, fingerprint, job_lock, private_directory
|
|
from monthly_reports.publishing import AtomicReportPublisher, OpenpyxlWorkbookBuilder
|
|
from monthly_reports.repository import DatabaseConfig as MonthlyConfig, PostgresReportRepository
|
|
from monthly_reports.service import MonthlyReportService
|
|
from monthly_reports.worker import MonthlyOutboxWorker, PostgresOutboxRepository
|
|
|
|
|
|
def create_instance(parent: Path, source: Path, expected_sha256: str, report_date: str,
|
|
*, prefix="arr-xml-replay-") -> Path:
|
|
if prefix not in {"arr-xml-replay-", "arr-api-simulation-"}:
|
|
raise ValueError("invalid_local_instance_prefix")
|
|
if parent.resolve().is_relative_to(PROJECT):
|
|
raise ValueError("replay_data_must_be_outside_repository")
|
|
root = Path(tempfile.mkdtemp(prefix=prefix, dir=parent)).resolve()
|
|
snapshot = NativeXMLSnapshot.create(root / "fixture", source, expected_sha256, report_date)
|
|
policy = load_processor_policy(PROJECT)
|
|
atomic_json(root / "instance.json", {
|
|
"version": "arr-local-replay-instance/v1", "root": str(root),
|
|
"source_manifest_sha256": fingerprint(snapshot.manifest),
|
|
"processor_version": policy.processor_version, "rule_set_sha256": policy.rule_set_sha256,
|
|
}, replace=False)
|
|
atomic_json(root / "login.json", {"username": "arr-replay", "password": secrets.token_urlsafe(24)}, replace=False)
|
|
return root
|
|
|
|
|
|
class ReplayRuntime:
|
|
def __init__(self, root, database, snapshot, policy, port, *, executor_factory=None,
|
|
portal_type=LocalReplayPortal, enable_upload=False, downloads_wrapper=SourceDateDownloads):
|
|
self.root, self.database, self.snapshot = root, database, snapshot
|
|
self.store = ManagedObjectStore(FilesystemObjectBackend(root / "objects", create=True))
|
|
self.repository = PostgresIngestionRepository(DatabaseConfig("local-replay"), connect=database.connect)
|
|
self.ingestion = IngestionService(DeliveryValidator(self.store, policy), self.repository)
|
|
processor = LocalDailyProcessor(policy)
|
|
if executor_factory is None:
|
|
self.executor = NativeXMLReplayExecutor(root / "handoffs", snapshot, policy, self.store,
|
|
self.repository, self.ingestion, processor)
|
|
else:
|
|
self.executor = executor_factory(root=root, snapshot=snapshot, policy=policy, object_store=self.store,
|
|
repository=self.repository, ingestion=self.ingestion, processor=processor)
|
|
self.queue = PersistentARRDownloads(root / "queue", self.executor)
|
|
try:
|
|
review = ProgrammaticUploadCoordinator(self.store, self.repository, self.ingestion,
|
|
processor, policy.processor_version, policy.rule_set_sha256)
|
|
config = MonthlyConfig("local-replay")
|
|
monthly_repository = PostgresReportRepository(config, connect=database.connect)
|
|
output = root / "outputs" / "monthly"
|
|
service = MonthlyReportService(monthly_repository, OpenpyxlWorkbookBuilder(),
|
|
AtomicReportPublisher(root, output, object_store=self.store),
|
|
output / ".staging")
|
|
self.worker = MonthlyOutboxWorker(PostgresOutboxRepository(config, connect=database.connect),
|
|
monthly_repository, service)
|
|
self.portal_repository = PostgresPortalRepository("local-replay", connect=database.connect)
|
|
app = PortalApplication(repository=self.portal_repository, price_reviews=review,
|
|
upload=review if enable_upload else None,
|
|
artifact_reader=ManagedObjectArtifactReader(self.store),
|
|
credentials=LoginCredentials(**document(root / "login.json")),
|
|
health=RuntimeHealth(database_ready=True, upload_ready=enable_upload, monthly_ready=True, download_ready=True),
|
|
arr_downloads=downloads_wrapper(self.queue, snapshot))
|
|
self.app = portal_type(app, snapshot, port)
|
|
self.stop = threading.Event()
|
|
self.thread = None
|
|
except BaseException:
|
|
self.queue.close(wait=True)
|
|
raise
|
|
|
|
def start_monthly_worker(self):
|
|
def run():
|
|
while not self.stop.is_set():
|
|
try:
|
|
result = self.worker.process_next()
|
|
if result.status != "idle":
|
|
atomic_json(self.root / "monthly-worker.json", {"status": result.status}, replace=True)
|
|
except Exception:
|
|
# Persist a fixed failure code; raw database/input values never enter a log.
|
|
atomic_json(self.root / "monthly-worker.json", {"status": "worker_error"}, replace=True)
|
|
self.stop.wait(1)
|
|
self.thread = threading.Thread(target=run, name="arr-local-monthly", daemon=True)
|
|
self.thread.start()
|
|
|
|
def close(self):
|
|
self.queue.close(wait=True)
|
|
self.stop.set()
|
|
if self.thread:
|
|
self.thread.join()
|
|
|
|
|
|
@contextmanager
|
|
def open_instance(root: Path, port: int, *, executor_factory=None, portal_type=LocalReplayPortal):
|
|
if not 1024 <= port <= 65535 or root.is_symlink():
|
|
raise ValueError("invalid_replay_instance")
|
|
root = root.resolve()
|
|
if root.is_relative_to(PROJECT):
|
|
raise ValueError("replay_data_must_be_outside_repository")
|
|
private_directory(root)
|
|
with job_lock(root):
|
|
if (root / "simulation.json").exists() != (executor_factory is not None):
|
|
raise ValueError("local_instance_mode_mismatch")
|
|
snapshot = NativeXMLSnapshot.load(root / "fixture")
|
|
policy = load_processor_policy(PROJECT)
|
|
expected = {"version": "arr-local-replay-instance/v1", "root": str(root),
|
|
"source_manifest_sha256": fingerprint(snapshot.manifest),
|
|
"processor_version": policy.processor_version, "rule_set_sha256": policy.rule_set_sha256}
|
|
if fingerprint(document(root / "instance.json")) != fingerprint(expected):
|
|
raise ValueError("replay_instance_identity_or_policy_changed")
|
|
database = ReplayDatabase(root)
|
|
try:
|
|
database.start()
|
|
runtime = ReplayRuntime(root, database, snapshot, policy, port,
|
|
executor_factory=executor_factory, portal_type=portal_type)
|
|
try:
|
|
yield runtime
|
|
finally:
|
|
runtime.close()
|
|
finally:
|
|
database.close()
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
actions = parser.add_subparsers(dest="action", required=True)
|
|
init = actions.add_parser("init", help="freeze one XML in a newly created private instance")
|
|
init.add_argument("--parent", type=Path, required=True)
|
|
init.add_argument("--xml", type=Path, required=True)
|
|
init.add_argument("--expected-sha256", required=True)
|
|
init.add_argument("--report-date", required=True)
|
|
run = actions.add_parser("serve", help="serve only this owned replay instance")
|
|
run.add_argument("--root", type=Path, required=True)
|
|
run.add_argument("--port", type=int, default=8873)
|
|
args = parser.parse_args(argv)
|
|
os.umask(0o077)
|
|
if args.action == "init":
|
|
root = create_instance(args.parent, args.xml, args.expected_sha256, args.report_date)
|
|
print(f"Local XML replay instance: {root}")
|
|
print(f"Private login credentials: {root / 'login.json'}")
|
|
return
|
|
def stop(_signal, _frame):
|
|
raise KeyboardInterrupt
|
|
signal.signal(signal.SIGTERM, stop)
|
|
try:
|
|
with open_instance(args.root, args.port) as runtime:
|
|
runtime.start_monthly_worker()
|
|
print(f"Local XML replay only: http://127.0.0.1:{args.port}/", flush=True)
|
|
serve(runtime.app, "127.0.0.1", args.port)
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|