126 lines
4.6 KiB
JavaScript
126 lines
4.6 KiB
JavaScript
import crypto from "node:crypto";
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { config as loadEnv } from "dotenv";
|
|
import OSS from "ali-oss";
|
|
import sharp from "sharp";
|
|
|
|
const root = process.cwd();
|
|
const extractPath = path.join(root, "docs", "research", "source-mobile-extract.json");
|
|
const manifestPath = path.join(root, "docs", "research", "source-assets.json");
|
|
|
|
loadEnv({ path: path.join(root, ".env") });
|
|
|
|
function endpointHost(endpoint) {
|
|
return endpoint.replace(/^https?:\/\//i, "").replace(/\/$/, "").split("/")[0];
|
|
}
|
|
|
|
function urlForKey(key) {
|
|
const configured = String(process.env.MEDIA_PUBLIC_BASE_URL ?? "").replace(/\/$/, "");
|
|
const endpoint = endpointHost(String(process.env.OSS_ENDPOINT ?? ""));
|
|
const baseUrl = configured || `https://${process.env.OSS_BUCKET_NAME}.${endpoint}`;
|
|
return `${baseUrl}/${key.split("/").map((part) => encodeURIComponent(part)).join("/")}`;
|
|
}
|
|
|
|
function buildClient() {
|
|
const endpoint = process.env.OSS_ENDPOINT;
|
|
const bucket = process.env.OSS_BUCKET_NAME;
|
|
const accessKeyId = process.env.OSS_ACCESS_KEY_ID;
|
|
const accessKeySecret = process.env.OSS_ACCESS_KEY_SECRET;
|
|
if (!endpoint || !bucket || !accessKeyId || !accessKeySecret) {
|
|
throw new Error("OSS 存储需要配置 OSS_ENDPOINT、OSS_BUCKET_NAME、OSS_ACCESS_KEY_ID 和 OSS_ACCESS_KEY_SECRET");
|
|
}
|
|
const host = endpointHost(endpoint);
|
|
return new OSS({
|
|
region: process.env.OSS_REGION ?? host.split(".")[0],
|
|
endpoint: /^https?:\/\//i.test(endpoint) ? endpoint : `https://${endpoint}`,
|
|
accessKeyId,
|
|
accessKeySecret,
|
|
bucket,
|
|
secure: true,
|
|
});
|
|
}
|
|
|
|
function extractCssUrls(value) {
|
|
if (!value) return [];
|
|
return [...String(value).matchAll(/url\((?:"|')?([^"')]+)(?:"|')?\)/g)].map((match) => match[1]);
|
|
}
|
|
|
|
const data = JSON.parse(await fs.readFile(extractPath, "utf8"));
|
|
const rawUrls = [
|
|
...data.extract.images.map((item) => item.src),
|
|
...data.extract.images.map((item) => item.src?.replace(/&/g, "&")),
|
|
...data.extract.backgroundImages.flatMap((item) => extractCssUrls(item.backgroundImage)),
|
|
...data.extract.links.filter((item) => item.as === "image").map((item) => item.href),
|
|
].filter(Boolean);
|
|
|
|
const uniqueUrls = [...new Set(rawUrls)]
|
|
.filter((url) => /^https?:\/\//.test(url))
|
|
.filter((url) => !url.includes("data:image"));
|
|
|
|
const manifest = [];
|
|
let cursor = 0;
|
|
const client = buildClient();
|
|
|
|
async function downloadOne(originalUrl, index) {
|
|
const hash = crypto.createHash("sha1").update(originalUrl).digest("hex").slice(0, 10);
|
|
try {
|
|
const response = await fetch(originalUrl, {
|
|
headers: {
|
|
"User-Agent":
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
|
|
Referer: "https://m.ctrip.com/",
|
|
},
|
|
});
|
|
if (!response.ok) {
|
|
manifest.push({ originalUrl, ok: false, status: response.status });
|
|
return;
|
|
}
|
|
const contentType = response.headers.get("content-type") ?? "";
|
|
const arrayBuffer = await response.arrayBuffer();
|
|
const sourceBuffer = Buffer.from(arrayBuffer);
|
|
const lossless = contentType.includes("png") || contentType.includes("svg");
|
|
const webpBuffer = await sharp(sourceBuffer)
|
|
.webp({ quality: 84, lossless, effort: 6 })
|
|
.toBuffer();
|
|
const filename = `asset-${String(index).padStart(3, "0")}-${hash}.webp`;
|
|
const objectKey = `wanqu/miniapp/static/source/${filename}`;
|
|
await client.put(objectKey, webpBuffer, {
|
|
headers: {
|
|
"Content-Type": "image/webp",
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
},
|
|
});
|
|
manifest.push({
|
|
originalUrl,
|
|
ok: true,
|
|
status: response.status,
|
|
contentType,
|
|
sourceBytes: arrayBuffer.byteLength,
|
|
bytes: webpBuffer.byteLength,
|
|
filename,
|
|
objectKey,
|
|
publicPath: urlForKey(objectKey),
|
|
});
|
|
} catch (error) {
|
|
manifest.push({ originalUrl, ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
}
|
|
}
|
|
|
|
async function worker() {
|
|
while (cursor < uniqueUrls.length) {
|
|
const index = cursor;
|
|
cursor += 1;
|
|
await downloadOne(uniqueUrls[index], index);
|
|
}
|
|
}
|
|
|
|
await Promise.all(Array.from({ length: 5 }, () => worker()));
|
|
manifest.sort((a, b) => (a.filename ?? "").localeCompare(b.filename ?? ""));
|
|
await fs.mkdir(path.dirname(manifestPath), { recursive: true });
|
|
await fs.writeFile(manifestPath, JSON.stringify({ generatedAt: new Date().toISOString(), assets: manifest }, null, 2));
|
|
|
|
const ok = manifest.filter((item) => item.ok).length;
|
|
const failed = manifest.length - ok;
|
|
console.log(`Downloaded ${ok}/${manifest.length} assets. Failed: ${failed}.`);
|