import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; const root = process.cwd(); const extractPath = path.join(root, "docs", "research", "source-mobile-extract.json"); const assetDir = path.join(root, "public", "assets", "source"); const manifestPath = path.join(assetDir, "manifest.json"); function extractCssUrls(value) { if (!value) return []; return [...String(value).matchAll(/url\((?:"|')?([^"')]+)(?:"|')?\)/g)].map((match) => match[1]); } function extensionFromContentType(contentType) { if (contentType.includes("webp")) return ".webp"; if (contentType.includes("png")) return ".png"; if (contentType.includes("gif")) return ".gif"; if (contentType.includes("svg")) return ".svg"; if (contentType.includes("jpeg") || contentType.includes("jpg")) return ".jpg"; return ".bin"; } function extensionFromUrl(url) { try { const parsed = new URL(url); const ext = path.extname(parsed.pathname).toLowerCase(); if ([".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg", ".avif"].includes(ext)) { return ext === ".jpeg" ? ".jpg" : ext; } } catch { return ""; } return ""; } const data = JSON.parse(await fs.readFile(extractPath, "utf8")); const rawUrls = [ ...data.extract.images.map((item) => item.src), ...data.extract.images.map((item) => item.src?.replace(/&/g, "&")), ...data.extract.backgroundImages.flatMap((item) => extractCssUrls(item.backgroundImage)), ...data.extract.links.filter((item) => item.as === "image").map((item) => item.href), ].filter(Boolean); const uniqueUrls = [...new Set(rawUrls)] .filter((url) => /^https?:\/\//.test(url)) .filter((url) => !url.includes("data:image")); await fs.mkdir(assetDir, { recursive: true }); const manifest = []; let cursor = 0; async function downloadOne(originalUrl, index) { const hash = crypto.createHash("sha1").update(originalUrl).digest("hex").slice(0, 10); try { const response = await fetch(originalUrl, { headers: { "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1", Referer: "https://m.ctrip.com/", }, }); if (!response.ok) { manifest.push({ originalUrl, ok: false, status: response.status }); return; } const contentType = response.headers.get("content-type") ?? ""; const ext = extensionFromUrl(originalUrl) || extensionFromContentType(contentType); const filename = `asset-${String(index).padStart(3, "0")}-${hash}${ext}`; const filePath = path.join(assetDir, filename); const arrayBuffer = await response.arrayBuffer(); await fs.writeFile(filePath, Buffer.from(arrayBuffer)); manifest.push({ originalUrl, ok: true, status: response.status, contentType, bytes: arrayBuffer.byteLength, filename, publicPath: `/assets/source/${filename}`, }); } catch (error) { manifest.push({ originalUrl, ok: false, error: error instanceof Error ? error.message : String(error) }); } } async function worker() { while (cursor < uniqueUrls.length) { const index = cursor; cursor += 1; await downloadOne(uniqueUrls[index], index); } } await Promise.all(Array.from({ length: 5 }, () => worker())); manifest.sort((a, b) => (a.filename ?? "").localeCompare(b.filename ?? "")); await fs.writeFile(manifestPath, JSON.stringify({ generatedAt: new Date().toISOString(), assets: manifest }, null, 2)); const ok = manifest.filter((item) => item.ok).length; const failed = manifest.length - ok; console.log(`Downloaded ${ok}/${manifest.length} assets. Failed: ${failed}.`);