103 lines
3.6 KiB
JavaScript
103 lines
3.6 KiB
JavaScript
import crypto from "node:crypto";
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
const root = process.cwd();
|
|
const extractPath = path.join(root, "docs", "research", "source-mobile-extract.json");
|
|
const assetDir = path.join(root, "public", "assets", "source");
|
|
const manifestPath = path.join(assetDir, "manifest.json");
|
|
|
|
function extractCssUrls(value) {
|
|
if (!value) return [];
|
|
return [...String(value).matchAll(/url\((?:"|')?([^"')]+)(?:"|')?\)/g)].map((match) => match[1]);
|
|
}
|
|
|
|
function extensionFromContentType(contentType) {
|
|
if (contentType.includes("webp")) return ".webp";
|
|
if (contentType.includes("png")) return ".png";
|
|
if (contentType.includes("gif")) return ".gif";
|
|
if (contentType.includes("svg")) return ".svg";
|
|
if (contentType.includes("jpeg") || contentType.includes("jpg")) return ".jpg";
|
|
return ".bin";
|
|
}
|
|
|
|
function extensionFromUrl(url) {
|
|
try {
|
|
const parsed = new URL(url);
|
|
const ext = path.extname(parsed.pathname).toLowerCase();
|
|
if ([".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg", ".avif"].includes(ext)) {
|
|
return ext === ".jpeg" ? ".jpg" : ext;
|
|
}
|
|
} catch {
|
|
return "";
|
|
}
|
|
return "";
|
|
}
|
|
|
|
const data = JSON.parse(await fs.readFile(extractPath, "utf8"));
|
|
const rawUrls = [
|
|
...data.extract.images.map((item) => item.src),
|
|
...data.extract.images.map((item) => item.src?.replace(/&/g, "&")),
|
|
...data.extract.backgroundImages.flatMap((item) => extractCssUrls(item.backgroundImage)),
|
|
...data.extract.links.filter((item) => item.as === "image").map((item) => item.href),
|
|
].filter(Boolean);
|
|
|
|
const uniqueUrls = [...new Set(rawUrls)]
|
|
.filter((url) => /^https?:\/\//.test(url))
|
|
.filter((url) => !url.includes("data:image"));
|
|
|
|
await fs.mkdir(assetDir, { recursive: true });
|
|
|
|
const manifest = [];
|
|
let cursor = 0;
|
|
|
|
async function downloadOne(originalUrl, index) {
|
|
const hash = crypto.createHash("sha1").update(originalUrl).digest("hex").slice(0, 10);
|
|
try {
|
|
const response = await fetch(originalUrl, {
|
|
headers: {
|
|
"User-Agent":
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
|
|
Referer: "https://m.ctrip.com/",
|
|
},
|
|
});
|
|
if (!response.ok) {
|
|
manifest.push({ originalUrl, ok: false, status: response.status });
|
|
return;
|
|
}
|
|
const contentType = response.headers.get("content-type") ?? "";
|
|
const ext = extensionFromUrl(originalUrl) || extensionFromContentType(contentType);
|
|
const filename = `asset-${String(index).padStart(3, "0")}-${hash}${ext}`;
|
|
const filePath = path.join(assetDir, filename);
|
|
const arrayBuffer = await response.arrayBuffer();
|
|
await fs.writeFile(filePath, Buffer.from(arrayBuffer));
|
|
manifest.push({
|
|
originalUrl,
|
|
ok: true,
|
|
status: response.status,
|
|
contentType,
|
|
bytes: arrayBuffer.byteLength,
|
|
filename,
|
|
publicPath: `/assets/source/${filename}`,
|
|
});
|
|
} catch (error) {
|
|
manifest.push({ originalUrl, ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
}
|
|
}
|
|
|
|
async function worker() {
|
|
while (cursor < uniqueUrls.length) {
|
|
const index = cursor;
|
|
cursor += 1;
|
|
await downloadOne(uniqueUrls[index], index);
|
|
}
|
|
}
|
|
|
|
await Promise.all(Array.from({ length: 5 }, () => worker()));
|
|
manifest.sort((a, b) => (a.filename ?? "").localeCompare(b.filename ?? ""));
|
|
await fs.writeFile(manifestPath, JSON.stringify({ generatedAt: new Date().toISOString(), assets: manifest }, null, 2));
|
|
|
|
const ok = manifest.filter((item) => item.ok).length;
|
|
const failed = manifest.length - ok;
|
|
console.log(`Downloaded ${ok}/${manifest.length} assets. Failed: ${failed}.`);
|