Greatest: serve catalog covers from a local copy (BL-0053)
The page hotlinked ~100 AniList CDN covers per load and some failed at random. The reader now downloads each catalog cover once (sequential, ~1/s, stops on 401/403/429, images up to 2 MB) into data/greatest-covers/ and serves it from /api/greatest-cover with a year-long browser cache; until a copy exists the route redirects to the catalog URL. The page also accepts the server's relative cover URLs, so owned titles show their library cover again. GREATEST_COVER_WARM=0 turns the downloads off (tests, offline installs). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cd6a30a76b
commit
e9da27e688
16 files changed
+277
-11
No files matched your search
@@ -0,0 +1,73 @@
|
||||
'use strict';
|
||||
|
||||
// Local copies of the Greatest catalog's cover images. Hotlinking ~100 covers per page
|
||||
// load makes the image CDN drop some of them at random; one polite download each fixes that.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
|
||||
const TYPES = { 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp', 'image/gif': '.gif', 'image/avif': '.avif' };
|
||||
const EXTENSIONS = new Set(Object.values(TYPES));
|
||||
const USER_AGENT = 'MangaCircus-Greatest/1.0';
|
||||
|
||||
class GreatestCovers {
|
||||
constructor(dir, { fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
|
||||
delayMs = 1000, maxBytes = 2 * 1024 * 1024, timeoutMs = 20000 } = {}) {
|
||||
this.dir = path.resolve(dir);
|
||||
Object.assign(this, { fetchImpl, sleep, delayMs, maxBytes, timeoutMs });
|
||||
this.files = new Map(); // key -> file name
|
||||
try {
|
||||
for (const name of fs.readdirSync(this.dir)) {
|
||||
const ext = path.extname(name);
|
||||
if (EXTENSIONS.has(ext)) this.files.set(path.basename(name, ext), name);
|
||||
}
|
||||
} catch (error) { if (error.code !== 'ENOENT') throw error; }
|
||||
}
|
||||
// A hash of the URL names the file, so a changed catalog URL is fetched again.
|
||||
version(url) { return crypto.createHash('sha256').update(String(url)).digest('hex').slice(0, 16); }
|
||||
file(url) {
|
||||
const name = url ? this.files.get(this.version(url)) : null;
|
||||
return name ? path.join(this.dir, name) : null;
|
||||
}
|
||||
// Sequential, paced, and stops on 403/429 like every other outbound client here.
|
||||
async warm(urls) {
|
||||
const result = { fetched: 0, skipped: 0, failed: 0, stopped: null };
|
||||
let requested = false;
|
||||
for (const url of new Set(urls.filter(url => /^https:\/\/\S+$/i.test(String(url || ''))))) {
|
||||
if (this.file(url)) { result.skipped++; continue; }
|
||||
if (requested) await this.sleep(this.delayMs);
|
||||
requested = true;
|
||||
try {
|
||||
await this.download(url);
|
||||
result.fetched++;
|
||||
} catch (error) {
|
||||
if (error.stop) { result.stopped = error.message; break; }
|
||||
result.failed++;
|
||||
}
|
||||
}
|
||||
// Duplicates and repeats of stored covers count as skipped.
|
||||
result.skipped += urls.filter(Boolean).length - new Set(urls.filter(Boolean)).size;
|
||||
return result;
|
||||
}
|
||||
async download(url) {
|
||||
const response = await this.fetchImpl(url, { headers: { 'user-agent': USER_AGENT },
|
||||
signal: AbortSignal.timeout(this.timeoutMs) });
|
||||
if ([401, 403, 429].includes(response.status)) throw Object.assign(new Error(`HTTP ${response.status}`), { stop: true });
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
const ext = TYPES[String(response.headers.get('content-type') || '').split(';')[0].trim().toLowerCase()];
|
||||
if (!ext) throw new Error('Not an image');
|
||||
if (Number(response.headers.get('content-length')) > this.maxBytes) throw new Error('Image too large');
|
||||
const body = Buffer.from(await response.arrayBuffer());
|
||||
if (!body.length || body.length > this.maxBytes) throw new Error('Image too large or empty');
|
||||
const key = this.version(url), name = key + ext;
|
||||
fs.mkdirSync(this.dir, { recursive: true });
|
||||
const temp = path.join(this.dir, `.${key}.${crypto.randomBytes(4).toString('hex')}.tmp`);
|
||||
try {
|
||||
fs.writeFileSync(temp, body);
|
||||
fs.renameSync(temp, path.join(this.dir, name));
|
||||
} finally { fs.rmSync(temp, { force: true }); }
|
||||
this.files.set(key, name);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { GreatestCovers };
|
||||
Reference in new issue
Block a user