Initial 智念AIGC platform
This commit is contained in:
inman
416
lib/server/storage.ts
Normal file
416
lib/server/storage.ts
Normal file
@@ -0,0 +1,416 @@
|
||||
import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
|
||||
import { extname, join } from "node:path";
|
||||
import { createAsset } from "@/lib/server/data-store";
|
||||
import { createId } from "@/lib/server/ids";
|
||||
import { resultDir, toAbsoluteUrl, uploadDir } from "@/lib/server/runtime";
|
||||
import type { Asset, AssetKind } from "@/lib/types";
|
||||
|
||||
type StoredObject = {
|
||||
url: string;
|
||||
storagePath: string;
|
||||
};
|
||||
|
||||
export async function saveUploadAsset(input: {
|
||||
ownerId: string;
|
||||
bytes: Buffer;
|
||||
fileName: string;
|
||||
contentType: string;
|
||||
origin: string;
|
||||
kind?: AssetKind;
|
||||
tags?: string[];
|
||||
}): Promise<Asset> {
|
||||
const stored = await storeBuffer({
|
||||
bytes: input.bytes,
|
||||
fileName: input.fileName,
|
||||
contentType: input.contentType,
|
||||
folder: "uploads",
|
||||
origin: input.origin
|
||||
});
|
||||
return createAsset({
|
||||
ownerId: input.ownerId,
|
||||
kind: input.kind || inferKind(input.contentType),
|
||||
name: input.fileName,
|
||||
url: stored.url,
|
||||
storagePath: stored.storagePath,
|
||||
source: "upload",
|
||||
tags: input.tags || [],
|
||||
metadata: {
|
||||
contentType: input.contentType,
|
||||
size: input.bytes.length
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function saveGeneratedAsset(input: {
|
||||
ownerId: string;
|
||||
bytes: Buffer;
|
||||
fileName: string;
|
||||
contentType: string;
|
||||
origin: string;
|
||||
source: Asset["source"];
|
||||
capability: string;
|
||||
kind?: AssetKind;
|
||||
jobId?: string;
|
||||
tags?: string[];
|
||||
metadata?: Record<string, unknown>;
|
||||
}): Promise<Asset> {
|
||||
const stored = await storeBuffer({
|
||||
bytes: input.bytes,
|
||||
fileName: input.fileName,
|
||||
contentType: input.contentType,
|
||||
folder: "generated-results",
|
||||
origin: input.origin
|
||||
});
|
||||
return createAsset({
|
||||
ownerId: input.ownerId,
|
||||
kind: input.kind || inferKind(input.contentType),
|
||||
name: input.fileName,
|
||||
url: stored.url,
|
||||
storagePath: stored.storagePath,
|
||||
source: input.source,
|
||||
tags: input.tags || [input.capability],
|
||||
metadata: {
|
||||
contentType: input.contentType,
|
||||
size: input.bytes.length,
|
||||
capability: input.capability,
|
||||
jobId: input.jobId,
|
||||
...(input.metadata || {})
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function importRemoteImageAsAsset(input: {
|
||||
ownerId: string;
|
||||
url: string;
|
||||
origin: string;
|
||||
source: Asset["source"];
|
||||
capability: string;
|
||||
jobId: string;
|
||||
index: number;
|
||||
}): Promise<Asset> {
|
||||
const response = await fetch(input.url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download generated image: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
const contentType = response.headers.get("content-type") || "image/png";
|
||||
const extension = extensionForContentType(contentType) || extname(new URL(input.url).pathname) || ".png";
|
||||
return saveGeneratedAsset({
|
||||
ownerId: input.ownerId,
|
||||
bytes,
|
||||
fileName: `${input.capability.replace(/\W+/g, "-")}-${input.jobId}-${input.index}${extension}`,
|
||||
contentType,
|
||||
origin: input.origin,
|
||||
source: input.source,
|
||||
capability: input.capability,
|
||||
jobId: input.jobId,
|
||||
metadata: {
|
||||
importedFrom: input.url
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function importRemoteAssetAsAsset(input: {
|
||||
ownerId: string;
|
||||
url: string;
|
||||
origin: string;
|
||||
source: Asset["source"];
|
||||
capability: string;
|
||||
jobId: string;
|
||||
index: number;
|
||||
fallbackContentType?: string;
|
||||
}): Promise<Asset> {
|
||||
const response = await fetch(input.url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download generated asset: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
const contentType = response.headers.get("content-type") || input.fallbackContentType || contentTypeForPath(new URL(input.url).pathname);
|
||||
const extension = extensionForContentType(contentType) || extname(new URL(input.url).pathname) || ".bin";
|
||||
return saveGeneratedAsset({
|
||||
ownerId: input.ownerId,
|
||||
bytes,
|
||||
fileName: `${input.capability.replace(/\W+/g, "-")}-${input.jobId}-${input.index}${extension}`,
|
||||
contentType,
|
||||
origin: input.origin,
|
||||
source: input.source,
|
||||
capability: input.capability,
|
||||
jobId: input.jobId,
|
||||
metadata: {
|
||||
importedFrom: input.url
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function saveMaskDataUrl(input: {
|
||||
ownerId: string;
|
||||
dataUrl: string;
|
||||
origin: string;
|
||||
jobHint?: string;
|
||||
}): Promise<Asset> {
|
||||
const parsed = parseDataUrl(input.dataUrl);
|
||||
return saveGeneratedAsset({
|
||||
ownerId: input.ownerId,
|
||||
bytes: parsed.bytes,
|
||||
fileName: `mask-${input.jobHint || createId("brush")}.png`,
|
||||
contentType: parsed.contentType,
|
||||
origin: input.origin,
|
||||
source: "edited",
|
||||
capability: "image.inpaint",
|
||||
kind: "mask",
|
||||
tags: ["mask"],
|
||||
metadata: {
|
||||
maskRule: "black keeps original pixels, white repaints selected pixels"
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function readLocalServedFile(area: "uploads" | "generated-results", pathParts: string[]): Promise<{
|
||||
bytes: Buffer;
|
||||
contentType: string;
|
||||
} | null> {
|
||||
const base = area === "uploads" ? uploadDir() : resultDir();
|
||||
const filePath = join(base, ...pathParts);
|
||||
try {
|
||||
return {
|
||||
bytes: await readFile(filePath),
|
||||
contentType: contentTypeForPath(filePath)
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readLegacyPublicFile(pathParts: string[]): Promise<{
|
||||
bytes: Buffer;
|
||||
contentType: string;
|
||||
} | null> {
|
||||
const filePath = join(process.cwd(), "runtime", "nianxx-play", "public", ...pathParts);
|
||||
try {
|
||||
return {
|
||||
bytes: await readFile(filePath),
|
||||
contentType: contentTypeForPath(filePath)
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteStoredAsset(asset: Asset): Promise<void> {
|
||||
if (!asset.storagePath) return;
|
||||
if (asset.storagePath.startsWith("uploads/") || asset.storagePath.startsWith("generated-results/")) {
|
||||
await deleteLocalStoredPath(asset.storagePath);
|
||||
return;
|
||||
}
|
||||
const oss = getOssConfig();
|
||||
if (!oss) return;
|
||||
const client = await createOssClient(oss);
|
||||
try {
|
||||
await (client as unknown as { delete: (name: string) => Promise<unknown> }).delete(asset.storagePath);
|
||||
} catch (error) {
|
||||
if (isMissingOssObject(error)) return;
|
||||
if (oss.fallbackEndpoint && isOssNetworkError(error)) {
|
||||
const fallbackClient = await createOssClient(oss, oss.fallbackEndpoint);
|
||||
await (fallbackClient as unknown as { delete: (name: string) => Promise<unknown> }).delete(asset.storagePath);
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function storeBuffer(input: {
|
||||
bytes: Buffer;
|
||||
fileName: string;
|
||||
contentType: string;
|
||||
folder: "uploads" | "generated-results";
|
||||
origin: string;
|
||||
}): Promise<StoredObject> {
|
||||
const oss = getOssConfig();
|
||||
const cleanName = sanitizeFileName(input.fileName);
|
||||
const objectKey = `${oss?.prefix || "zhinian"}/${input.folder}/${new Date().toISOString().slice(0, 10)}/${createId("file")}-${cleanName}`;
|
||||
if (oss) {
|
||||
const client = await createOssClient(oss);
|
||||
try {
|
||||
await client.put(objectKey, input.bytes, {
|
||||
mime: input.contentType
|
||||
});
|
||||
} catch (error) {
|
||||
if (!oss.fallbackEndpoint || !isOssNetworkError(error)) throw error;
|
||||
const fallbackClient = await createOssClient(oss, oss.fallbackEndpoint);
|
||||
await fallbackClient.put(objectKey, input.bytes, {
|
||||
mime: input.contentType
|
||||
});
|
||||
}
|
||||
await makeOssObjectPublic(client, objectKey, oss);
|
||||
const baseUrl = oss.publicBaseUrl.replace(/\/$/, "");
|
||||
return {
|
||||
url: `${baseUrl}/${objectKey}`,
|
||||
storagePath: objectKey
|
||||
};
|
||||
}
|
||||
|
||||
const base = input.folder === "uploads" ? uploadDir() : resultDir();
|
||||
const relativePath = join(new Date().toISOString().slice(0, 10), `${createId("file")}-${cleanName}`);
|
||||
const absolutePath = join(base, relativePath);
|
||||
await mkdir(join(base, new Date().toISOString().slice(0, 10)), { recursive: true });
|
||||
await writeFile(absolutePath, input.bytes);
|
||||
const publicPath = `/${input.folder}/${relativePath.split(/[\\/]/).map(encodeURIComponent).join("/")}`;
|
||||
return {
|
||||
url: toAbsoluteUrl(publicPath, input.origin),
|
||||
storagePath: `${input.folder}/${relativePath}`
|
||||
};
|
||||
}
|
||||
|
||||
function getOssConfig() {
|
||||
const endpoint = process.env.ALI_OSS_ENDPOINT;
|
||||
const bucket = process.env.ALI_OSS_BUCKET;
|
||||
const accessKeyId = process.env.ALI_OSS_ACCESS_KEY_ID;
|
||||
const accessKeySecret = process.env.ALI_OSS_ACCESS_KEY_SECRET;
|
||||
const publicBaseUrl = process.env.ALI_OSS_PUBLIC_BASE_URL;
|
||||
if (!endpoint || !bucket || !accessKeyId || !accessKeySecret || !publicBaseUrl) return null;
|
||||
const publicEndpoint = inferPublicEndpoint(publicBaseUrl);
|
||||
const preferredEndpoint = preferPublicOssEndpoint(endpoint, publicEndpoint) ? publicEndpoint : endpoint;
|
||||
const fallbackEndpoint = preferredEndpoint === endpoint ? publicEndpoint : endpoint;
|
||||
return {
|
||||
endpoint: preferredEndpoint || endpoint,
|
||||
bucket,
|
||||
accessKeyId,
|
||||
accessKeySecret,
|
||||
publicBaseUrl: normalizeOssPublicBaseUrl(publicBaseUrl, bucket),
|
||||
fallbackEndpoint: fallbackEndpoint === preferredEndpoint ? undefined : fallbackEndpoint,
|
||||
prefix: process.env.ALI_OSS_PREFIX || "zhinian",
|
||||
region: inferOssRegion(endpoint)
|
||||
};
|
||||
}
|
||||
|
||||
async function createOssClient(oss: NonNullable<ReturnType<typeof getOssConfig>>, endpoint = oss.endpoint) {
|
||||
const { default: OSS } = await import("ali-oss");
|
||||
return new OSS({
|
||||
region: oss.region,
|
||||
endpoint,
|
||||
bucket: oss.bucket,
|
||||
accessKeyId: oss.accessKeyId,
|
||||
accessKeySecret: oss.accessKeySecret,
|
||||
timeout: 8000
|
||||
});
|
||||
}
|
||||
|
||||
async function makeOssObjectPublic(client: Awaited<ReturnType<typeof createOssClient>>, objectKey: string, oss: NonNullable<ReturnType<typeof getOssConfig>>) {
|
||||
try {
|
||||
await (client as unknown as { putACL: (name: string, acl: string) => Promise<unknown> }).putACL(objectKey, "public-read");
|
||||
} catch (error) {
|
||||
if (!oss.fallbackEndpoint || !isOssNetworkError(error)) throw error;
|
||||
const fallbackClient = await createOssClient(oss, oss.fallbackEndpoint);
|
||||
await (fallbackClient as unknown as { putACL: (name: string, acl: string) => Promise<unknown> }).putACL(objectKey, "public-read");
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteLocalStoredPath(storagePath: string) {
|
||||
const [area, ...parts] = storagePath.split(/[\\/]/);
|
||||
if (!parts.length) return;
|
||||
const base = area === "uploads" ? uploadDir() : area === "generated-results" ? resultDir() : null;
|
||||
if (!base) return;
|
||||
try {
|
||||
await unlink(join(base, ...parts));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function inferOssRegion(endpoint: string) {
|
||||
try {
|
||||
const host = new URL(endpoint).hostname;
|
||||
const match = host.match(/^oss-([a-z0-9-]+?)(?:-internal)?\./);
|
||||
return match ? `oss-${match[1]}` : undefined;
|
||||
} catch {
|
||||
const match = endpoint.match(/oss-([a-z0-9-]+?)(?:-internal)?(?:\.|$)/);
|
||||
return match ? `oss-${match[1]}` : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOssPublicBaseUrl(publicBaseUrl: string, bucket: string) {
|
||||
try {
|
||||
const url = new URL(publicBaseUrl);
|
||||
if (url.hostname.startsWith("oss-") && url.pathname === "/") {
|
||||
url.hostname = `${bucket}.${url.hostname}`;
|
||||
}
|
||||
return url.toString().replace(/\/$/, "");
|
||||
} catch {
|
||||
return publicBaseUrl.replace(/\/$/, "");
|
||||
}
|
||||
}
|
||||
|
||||
function inferPublicEndpoint(publicBaseUrl: string) {
|
||||
try {
|
||||
const url = new URL(publicBaseUrl);
|
||||
if (url.hostname.startsWith("oss-")) return url.origin;
|
||||
const match = url.hostname.match(/^[^.]+\.(oss-[^.]+\.aliyuncs\.com)$/);
|
||||
return match ? `${url.protocol}//${match[1]}` : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function preferPublicOssEndpoint(endpoint: string, publicEndpoint?: string) {
|
||||
if (!publicEndpoint || process.env.ALI_OSS_USE_INTERNAL_ENDPOINT === "1") return false;
|
||||
try {
|
||||
return new URL(endpoint).hostname.includes("-internal.");
|
||||
} catch {
|
||||
return endpoint.includes("-internal.");
|
||||
}
|
||||
}
|
||||
|
||||
function isMissingOssObject(error: unknown) {
|
||||
return typeof error === "object" && error !== null && (
|
||||
"code" in error && String((error as { code?: unknown }).code) === "NoSuchKey" ||
|
||||
"status" in error && Number((error as { status?: unknown }).status) === 404
|
||||
);
|
||||
}
|
||||
|
||||
function isOssNetworkError(error: unknown) {
|
||||
if (typeof error !== "object" || error === null) return false;
|
||||
const code = "code" in error ? String((error as { code?: unknown }).code) : "";
|
||||
const message = "message" in error ? String((error as { message?: unknown }).message) : "";
|
||||
return ["ENOTFOUND", "ETIMEDOUT", "ECONNREFUSED", "EAI_AGAIN"].includes(code) || /network|timeout|ENOTFOUND|ETIMEDOUT/i.test(message);
|
||||
}
|
||||
|
||||
function sanitizeFileName(fileName: string): string {
|
||||
const fallback = "asset.bin";
|
||||
const clean = fileName.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
return clean || fallback;
|
||||
}
|
||||
|
||||
function inferKind(contentType: string): AssetKind {
|
||||
if (contentType.startsWith("image/")) return "image";
|
||||
if (contentType.startsWith("video/")) return "video";
|
||||
return "other";
|
||||
}
|
||||
|
||||
function parseDataUrl(dataUrl: string): { contentType: string; bytes: Buffer } {
|
||||
const match = dataUrl.match(/^data:([^;,]+);base64,(.+)$/);
|
||||
if (!match) throw new Error("Invalid mask data URL.");
|
||||
return {
|
||||
contentType: match[1],
|
||||
bytes: Buffer.from(match[2], "base64")
|
||||
};
|
||||
}
|
||||
|
||||
function extensionForContentType(contentType: string): string {
|
||||
if (contentType.includes("jpeg") || contentType.includes("jpg")) return ".jpg";
|
||||
if (contentType.includes("webp")) return ".webp";
|
||||
if (contentType.includes("svg")) return ".svg";
|
||||
if (contentType.includes("mp4")) return ".mp4";
|
||||
if (contentType.includes("quicktime")) return ".mov";
|
||||
if (contentType.includes("webm")) return ".webm";
|
||||
return ".png";
|
||||
}
|
||||
|
||||
function contentTypeForPath(filePath: string): string {
|
||||
const ext = extname(filePath).toLowerCase();
|
||||
if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg";
|
||||
if (ext === ".png") return "image/png";
|
||||
if (ext === ".webp") return "image/webp";
|
||||
if (ext === ".svg") return "image/svg+xml";
|
||||
if (ext === ".mp4") return "video/mp4";
|
||||
return "application/octet-stream";
|
||||
}
|
||||
Reference in New Issue
Block a user