From e9da27e6883324765ffbbd59f03788319869634e Mon Sep 17 00:00:00 2001 From: maashkinoozhe Date: Mon, 26 Oct 2026 00:38:55 +0100 Subject: [PATCH] Greatest: serve catalog covers from a local copy (BL-0053) The page hotlinked ~100 AniList CDN covers per load and some failed at random. The reader now downloads each catalog cover once (sequential, ~1/s, stops on 401/403/429, images up to 2 MB) into data/greatest-covers/ and serves it from /api/greatest-cover with a year-long browser cache; until a copy exists the route redirects to the catalog URL. The page also accepts the server's relative cover URLs, so owned titles show their library cover again. GREATEST_COVER_WARM=0 turns the downloads off (tests, offline installs). Co-Authored-By: Claude Opus 5.5 --- docker-compose.yml | 1 + docs/architecture.md | 1 + docs/deployment.md | 3 +- docs/greatest.md | 2 +- greatest/covers.js | 73 +++++++++++++++++++++++++++ public/greatest.js | 6 ++- server.js | 29 ++++++++++- test/admin-role.test.js | 3 +- test/backup.test.js | 2 +- test/discovery-server.test.js | 2 +- test/greatest-cover-route.test.js | 75 +++++++++++++++++++++++++++ test/greatest-covers.test.js | 84 +++++++++++++++++++++++++++++++ test/library-index.test.js | 2 +- test/services.test.js | 2 +- tools/test-compose.js | 1 + tools/test-greatest-ui.js | 2 +- 16 files changed, 277 insertions(+), 11 deletions(-) create mode 100644 greatest/covers.js create mode 100644 test/greatest-cover-route.test.js create mode 100644 test/greatest-covers.test.js diff --git a/docker-compose.yml b/docker-compose.yml index 03eb556..ffe7d02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,6 +60,7 @@ services: SCRAPER_TOKEN_FILE: /run/secrets/scraper_token DISCOVERY_HARVEST_AT: ${DISCOVERY_HARVEST_AT:-03:30} DISCOVERY_ANILIST_URL: ${DISCOVERY_ANILIST_URL:-} + GREATEST_COVER_WARM: ${GREATEST_COVER_WARM:-1} networks: [backend, egress] healthcheck: test: [CMD, node, tools/healthcheck.js, reader] diff --git a/docs/architecture.md b/docs/architecture.md index 4639631..139df65 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -207,6 +207,7 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js | `POST /api/admin/user/role` `{name, admin}` | admin | Grant or revoke admin rights; keeps the password and sessions. Revoking your own rights is refused, so one admin always remains | | `POST /api/admin/user/delete` `{name}` | admin | Delete a user and their data | | `GET /api/admin/backup` · `POST /api/admin/backup` `{command:"settings"\|"run"}` | admin | Built-in backup status, schedule/retention, "back up now" ([backup.md](backup.md)) | +| `GET /api/greatest-cover?id=&v=` | user | The Greatest entry's cover from `data/greatest-covers/` (`greatest/covers.js`), cached by the browser for a year; `v` changes with the catalog URL. A `302` to the catalog URL until the copy exists. `/api/greatest` lists this URL as `cover` for titles not in the library | | `GET /api/discovery` | user | The caller's discovery feed, settings, feedback and `refresh: { state: idle\|queued\|running, position?, nextAllowedAt?, blockedReason? }` (§6.6) | | `POST /api/discovery/feedback` `{candidateId, value}` | user | The caller's choice for a candidate; requires opt-in | | `POST /api/discovery/refresh` | user | Queue a per-user AniList harvest for the caller (opt-in required, else `403`). `{ harvestId, position }`; `409` while one is queued or running for the caller; `429 { nextAllowedAt }` within an hour of the last accepted refresh; `503 { blockedReason }` after AniList pushed back, until a nightly or admin harvest | diff --git a/docs/deployment.md b/docs/deployment.md index 6829333..e69d0ce 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -21,7 +21,7 @@ Data on the host, all bind mounts (no named volumes): | Path (`.env`) | Contents | Back up? | |---|---|---| | `LIBRARY_PATH` | The manga: `Manga/Book/Chapter/images` ([library-format.md](library-format.md)) | Yes, it is your collection | -| `DATA_PATH` | `db.json` (users, progress, votes), catalog, discovery feeds, library index cache | Yes (built-in backup, section 8). `discovery-cache/` inside it is safe to delete and may be left out of backups: the next harvest rebuilds it | +| `DATA_PATH` | `db.json` (users, progress, votes), catalog, discovery feeds, library index cache | Yes (built-in backup, section 8). `discovery-cache/` inside it is safe to delete and may be left out of backups: the next harvest rebuilds it. The same holds for `greatest-covers/` (local copies of the Greatest catalog's cover images, fetched again after the next start) | | `SCRAPER_DATA_PATH` | Queue, settings, source ledger, metadata cache | Included in the built-in backup | | `BACKUP_PATH` | Built-in backup archives | Keep it on another disk or NAS | | `SCRAPER_TOKEN_PATH` | Shared secret between reader and scraper | Regenerate if lost | @@ -71,6 +71,7 @@ demo manga in an empty library after the build in section 5: | `READING_WINDOWS_JSON`, `READING_TIMEZONE`, `READING_QUIZ_*` | off | Optional server-enforced reading hours ([reading-limiter.md](reading-limiter.md)) | | `DISCOVERY_HARVEST_AT` | `03:30` | Daily time (`HH:MM`) of the nightly Discover harvest from AniList, in the container's local time zone (`TZ`). Only users who opted in are harvested; the admin can change the time or switch it off on the Discover page ([discovery.md](discovery.md)) | | `DISCOVERY_ANILIST_URL` | AniList's public API | Tests only: points the discovery harvest at a fixture server. Leave unset in production | +| `GREATEST_COVER_WARM` | `1` | `0` stops the reader from downloading the Greatest catalog's covers (about 100 images, one per second, 10 s after start and once a day for missing ones). Tests and offline installs set `0`; the page then loads covers from the catalog's own URLs | | `APP_IMAGE`, `VCS_REF` | `mangacircus:local`, `development` | Image to run; set a pinned registry image to deploy prebuilt releases (section 7) | | `COMPOSE_FILE` | `docker-compose.yml` | Extra Compose files, `:`-separated (e.g. `docker-compose.yml:deploy/compose.https.yml`) | | `MANGACIRCUS_DOMAIN` | – | Host name for `deploy/compose.https.yml` | diff --git a/docs/greatest.md b/docs/greatest.md index ab89334..919f327 100644 --- a/docs/greatest.md +++ b/docs/greatest.md @@ -23,7 +23,7 @@ Within each cohort, **Rank by my vote** sorts flame → good → neutral → sku The one-time seed is curated from the research document. All 106 catalog entries now have a cover URL. Most were matched to public AniList manga metadata; the five initial gaps were checked individually. Phoenix (1954) and Nausicaä use matching AniList records; Kitarō uses Drawn & Quarterly's Kitaro edition, Someone Hertz uses Shonen Jump's first-volume cover, and Hitoner uses Shueisha's 2026 first-volume cover. Hitoner's AniList 2024 entry represents an earlier work, so it was **not** used for the 2026 series. The five added URLs returned image content on 2026-10-09, but remote hosts can change them; the UI still shows a title-letter fallback if an image fails. -`node tools/enrich-greatest-catalog.js` is an **optional manual** public-AniList metadata refresh, paced at one request every 1.5 seconds. It only accepts matching manga title and **original serialization year**, leaving uncertain results untouched rather than using a sequel or remake cover. The dashboard makes **no metadata calls when loading**; only an admin pressing **Search MangaDex** triggers a bounded lookup. Browser covers are lazy-loaded from their listed source; local library covers take precedence. +`node tools/enrich-greatest-catalog.js` is an **optional manual** public-AniList metadata refresh, paced at one request every 1.5 seconds. It only accepts matching manga title and **original serialization year**, leaving uncertain results untouched rather than using a sequel or remake cover. The dashboard makes **no metadata calls when loading**; only an admin pressing **Search MangaDex** triggers a bounded lookup. Local library covers take precedence. For the rest, the reader keeps its own copy of each catalog cover in `data/greatest-covers/` (`greatest/covers.js`): it downloads missing ones one at a time, about one a second, 10 seconds after start and once a day, stops on HTTP 401/403/429, and accepts only JPEG/PNG/WebP/GIF/AVIF images up to 2 MB from the catalog's own HTTPS URLs. The page loads covers from `/api/greatest-cover`, which the browser caches for a year; until a copy exists that route redirects to the catalog URL. Hotlinking about 100 covers per page load made the image CDN drop some of them at random. `GREATEST_COVER_WARM=0` switches the downloads off. Public catalog chapter counts are often absent; the UI says “Chapters unknown” rather than inventing one. For owned titles, the local library count wins. Status comes from AniList's releasing/finished flag or local library metadata, and may be unknown. Official source links may be publisher editions, chapter services, or an award/info lead; they do not imply all chapters are available in every country. No unverified hoster coverage is claimed. diff --git a/greatest/covers.js b/greatest/covers.js new file mode 100644 index 0000000..e0e27af --- /dev/null +++ b/greatest/covers.js @@ -0,0 +1,73 @@ +'use strict'; + +// Local copies of the Greatest catalog's cover images. Hotlinking ~100 covers per page +// load makes the image CDN drop some of them at random; one polite download each fixes that. +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); + +const TYPES = { 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp', 'image/gif': '.gif', 'image/avif': '.avif' }; +const EXTENSIONS = new Set(Object.values(TYPES)); +const USER_AGENT = 'MangaCircus-Greatest/1.0'; + +class GreatestCovers { + constructor(dir, { fetchImpl = fetch, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), + delayMs = 1000, maxBytes = 2 * 1024 * 1024, timeoutMs = 20000 } = {}) { + this.dir = path.resolve(dir); + Object.assign(this, { fetchImpl, sleep, delayMs, maxBytes, timeoutMs }); + this.files = new Map(); // key -> file name + try { + for (const name of fs.readdirSync(this.dir)) { + const ext = path.extname(name); + if (EXTENSIONS.has(ext)) this.files.set(path.basename(name, ext), name); + } + } catch (error) { if (error.code !== 'ENOENT') throw error; } + } + // A hash of the URL names the file, so a changed catalog URL is fetched again. + version(url) { return crypto.createHash('sha256').update(String(url)).digest('hex').slice(0, 16); } + file(url) { + const name = url ? this.files.get(this.version(url)) : null; + return name ? path.join(this.dir, name) : null; + } + // Sequential, paced, and stops on 403/429 like every other outbound client here. + async warm(urls) { + const result = { fetched: 0, skipped: 0, failed: 0, stopped: null }; + let requested = false; + for (const url of new Set(urls.filter(url => /^https:\/\/\S+$/i.test(String(url || ''))))) { + if (this.file(url)) { result.skipped++; continue; } + if (requested) await this.sleep(this.delayMs); + requested = true; + try { + await this.download(url); + result.fetched++; + } catch (error) { + if (error.stop) { result.stopped = error.message; break; } + result.failed++; + } + } + // Duplicates and repeats of stored covers count as skipped. + result.skipped += urls.filter(Boolean).length - new Set(urls.filter(Boolean)).size; + return result; + } + async download(url) { + const response = await this.fetchImpl(url, { headers: { 'user-agent': USER_AGENT }, + signal: AbortSignal.timeout(this.timeoutMs) }); + if ([401, 403, 429].includes(response.status)) throw Object.assign(new Error(`HTTP ${response.status}`), { stop: true }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const ext = TYPES[String(response.headers.get('content-type') || '').split(';')[0].trim().toLowerCase()]; + if (!ext) throw new Error('Not an image'); + if (Number(response.headers.get('content-length')) > this.maxBytes) throw new Error('Image too large'); + const body = Buffer.from(await response.arrayBuffer()); + if (!body.length || body.length > this.maxBytes) throw new Error('Image too large or empty'); + const key = this.version(url), name = key + ext; + fs.mkdirSync(this.dir, { recursive: true }); + const temp = path.join(this.dir, `.${key}.${crypto.randomBytes(4).toString('hex')}.tmp`); + try { + fs.writeFileSync(temp, body); + fs.renameSync(temp, path.join(this.dir, name)); + } finally { fs.rmSync(temp, { force: true }); } + this.files.set(key, name); + } +} + +module.exports = { GreatestCovers }; diff --git a/public/greatest.js b/public/greatest.js index ce06802..ad75836 100644 --- a/public/greatest.js +++ b/public/greatest.js @@ -65,8 +65,10 @@ function cover(item) { const box = h('span', { class: 'greatest-cover' }, h('span', { class: 'greatest-cover-fallback', 'aria-hidden': 'true' }, item.title.slice(0, 1))); - if (item.cover && safeHref(item.cover)) { - const image = h('img', { src: item.cover, alt: '', loading: 'lazy' }); + // Covers come from the server (cached catalog covers, owned library covers) as relative URLs. + const src = /^(api\/greatest-cover\?|lib\/)/.test(item.cover || '') ? item.cover : safeHref(item.cover); + if (src) { + const image = h('img', { src, alt: '', loading: 'lazy' }); image.onerror = () => image.remove(); box.append(image); } diff --git a/server.js b/server.js index 242f203..9a2b5e5 100644 --- a/server.js +++ b/server.js @@ -19,6 +19,7 @@ const { Catalog, CatalogError, collectionViews, TYPES, ROLES } = require('./cata const { RATINGS: GREATEST_RATINGS, loadCatalog: loadGreatestCatalog, findOwned: findGreatestOwned, countsFor: greatestCounts, scoreFor: greatestScore } = require('./greatest'); const { hosterLinks } = require('./greatest/hosters'); +const { GreatestCovers } = require('./greatest/covers'); const { secret } = require('./runtime/config'); const { PeeringState, requirePeer } = require('./peering/state'); const peerClient = require('./peering/client'); @@ -65,6 +66,10 @@ fs.mkdirSync(DATA, { recursive: true }); const catalog = new Catalog(DATA); const greatestCatalog = loadGreatestCatalog(); const greatestById = new Map(greatestCatalog.entries.map(entry => [entry.id, entry])); +const greatestCovers = new GreatestCovers(path.join(DATA, 'greatest-covers')); +// The versioned URL lets browsers keep a cover for a year; a new catalog URL changes `v`. +const greatestCoverUrl = entry => entry.cover + ? `api/greatest-cover?id=${encodeURIComponent(entry.id)}&v=${greatestCovers.version(entry.cover)}` : null; const DB_FILE = path.join(DATA, 'db.json'), CACHE_FILE = path.join(DATA, 'scan-cache.json'), INDEX_FILE = path.join(DATA, 'library-index.json'); const db = { users: {}, sessions: {}, progress: {}, reading: {}, readingAccess: {}, follows: {}, notes: {}, votes: {}, greatestRatings: {}, discoveryFeedback: {}, known: {} }; // A corrupt db.json stops the server instead of silently starting empty and overwriting it. @@ -620,7 +625,7 @@ const routes = { const counts = greatestCounts(entry.id, db.greatestRatings || {}, db.users); return { ...entry, - cover: owned?.cover ? url(owned.cover) : entry.cover, + cover: owned?.cover ? url(owned.cover) : greatestCoverUrl(entry), chapters: owned ? owned.chapters.length : entry.chapters, status: owned?.meta?.status || entry.status, library: owned ? { id: owned.id, chapters: owned.chapters.length } : null, @@ -1034,6 +1039,16 @@ http.createServer(async (req, res) => { if (!c?.customCover) bad(404, 'Cover not found'); return serveFile(req, res, path.join(DATA, 'collection-covers'), c.customCover, 'private, no-cache', IMG); } + if (pathname === '/api/greatest-cover' && req.method === 'GET') { + if (!u) bad(401, 'Please log in'); + const entry = greatestById.get(searchParams.get('id')); + if (!entry?.cover) bad(404, 'Cover not found'); + const file = greatestCovers.file(entry.cover); + if (file) return serveFile(req, res, greatestCovers.dir, path.basename(file), 'private, max-age=31536000, immutable', IMG); + // Not downloaded yet: the browser fetches the catalog's own URL this once. + res.writeHead(302, { Location: entry.cover, 'Cache-Control': 'no-store' }); + return res.end(); + } if (pathname === '/api/peering/cover' && req.method === 'GET') { if (!u) bad(401, 'Please log in'); admin(u); @@ -1074,6 +1089,18 @@ http.createServer(async (req, res) => { if (lib.mangas.length) refreshChanged(); else rescan(); if (SCAN_MINUTES > 0) setInterval(() => refreshChanged(), SCAN_MINUTES * 60000).unref(); +// Fetch missing Greatest covers shortly after start and retry failures once a day; +// covers already on disk cost nothing. +async function warmGreatestCovers() { + try { + const result = await greatestCovers.warm(greatestCatalog.entries.map(entry => entry.cover)); + if (result.fetched || result.failed || result.stopped) console.log('[greatest] covers', JSON.stringify(result)); + } catch (error) { console.error('[greatest] covers', error.message); } +} +if (process.env.GREATEST_COVER_WARM !== '0') { + setTimeout(warmGreatestCovers, 10000).unref(); + setInterval(warmGreatestCovers, 86400000).unref(); +} // Persist the last reconciliation time. Restarting a healthy reader should not // trigger another full tree walk; overdue reconciliations run within a minute. if (FULL_SCAN_MINUTES > 0) setInterval(() => { diff --git a/test/admin-role.test.js b/test/admin-role.test.js index 6f5f386..64e4237 100644 --- a/test/admin-role.test.js +++ b/test/admin-role.test.js @@ -21,7 +21,8 @@ test('only admins can promote or demote, nobody can demote themselves, and chang const port = await freePort(), output = []; child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true, env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0', - SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '' } }); + SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '', + GREATEST_COVER_WARM: '0' } }); child.stdout.on('data', d => output.push(String(d))); child.stderr.on('data', d => output.push(String(d))); const base = `http://127.0.0.1:${port}`; diff --git a/test/backup.test.js b/test/backup.test.js index a49f9b5..2559483 100644 --- a/test/backup.test.js +++ b/test/backup.test.js @@ -251,7 +251,7 @@ test('reader and scraper: admin backup API pauses and resumes the real queue', { const base = `http://127.0.0.1:${rp}`, token = 'backup-test-only'; const env = { ...process.env, DATA: f.data, LIBRARY: library, SCRAPER_DATA: path.join(f.data, 'scraper'), BACKUP: f.target, PORT: String(rp), SCRAPER_PORT: String(sp), SCRAPER_HOST: '127.0.0.1', SCRAPER_URL: `http://127.0.0.1:${sp}`, SENTINEL_URL: 'http://127.0.0.1:9', - SCRAPER_TOKEN: token, SCRAPER_TOKEN_FILE: '', SCAN_INTERVAL: '0' }; + SCRAPER_TOKEN: token, SCRAPER_TOKEN_FILE: '', SCAN_INTERVAL: '0', GREATEST_COVER_WARM: '0' }; for (const script of ['server.js', 'scraper/daemon.js']) { const child = spawn(process.execPath, [script], { cwd: root, env, windowsHide: true }); children.push(child); for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => output.push(chunk.toString())); diff --git a/test/discovery-server.test.js b/test/discovery-server.test.js index e85b2ed..6609893 100644 --- a/test/discovery-server.test.js +++ b/test/discovery-server.test.js @@ -81,7 +81,7 @@ async function startReader({ anilist, discovery = {}, mangaMeta = null, scraperU const child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true, env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0', SCRAPER_URL: scraperUrl, SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '', - DISCOVERY_ANILIST_URL: anilist ? anilist.url : 'http://127.0.0.1:9/', DISCOVERY_HARVEST_AT: '' } }); + DISCOVERY_ANILIST_URL: anilist ? anilist.url : 'http://127.0.0.1:9/', DISCOVERY_HARVEST_AT: '', GREATEST_COVER_WARM: '0' } }); child.stdout.on('data', d => output.push(String(d))); child.stderr.on('data', d => output.push(String(d))); const base = `http://127.0.0.1:${port}`; diff --git a/test/greatest-cover-route.test.js b/test/greatest-cover-route.test.js new file mode 100644 index 0000000..c2b6b4f --- /dev/null +++ b/test/greatest-cover-route.test.js @@ -0,0 +1,75 @@ +'use strict'; +// BL-0053: Greatest covers are served from the reader's own copy, with a redirect until one exists. +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('fs'), os = require('os'), path = require('path'), net = require('net'), crypto = require('crypto'); +const { spawn } = require('child_process'); +const { loadCatalog } = require('../greatest'); +const { GreatestCovers } = require('../greatest/covers'); + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const hashPw = pw => { const salt = crypto.randomBytes(16).toString('hex'); return salt + ':' + crypto.scryptSync(pw, salt, 32).toString('hex'); }; +const sha = s => crypto.createHash('sha256').update(s).digest('hex'); +const PNG = Buffer.from('89504e470d0a1a0a0000000d49484452', 'hex'); + +async function freePort() { + const listener = net.createServer(); + await new Promise(resolve => listener.listen(0, '127.0.0.1', resolve)); + const port = listener.address().port; + await new Promise(resolve => listener.close(resolve)); + return port; +} + +test('Greatest covers come from the local copy, else redirect to the catalog URL', async t => { + const entries = loadCatalog().entries.filter(entry => entry.cover); + const [cached, missing] = entries; + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-greatest-cover-')); + const library = path.join(root, 'library'), data = path.join(root, 'data'); + fs.mkdirSync(library, { recursive: true }); + fs.mkdirSync(path.join(data, 'greatest-covers'), { recursive: true }); + const version = new GreatestCovers(path.join(data, 'greatest-covers')).version(cached.cover); + fs.writeFileSync(path.join(data, 'greatest-covers', `${version}.png`), PNG); + const token = crypto.randomBytes(32).toString('base64url'); + fs.writeFileSync(path.join(data, 'db.json'), JSON.stringify({ + users: { reader: { name: 'reader', hash: hashPw('reader-password'), admin: false, created: Date.now(), settings: {} } }, + sessions: { [sha(token)]: { u: 'reader', t: Date.now() } } })); + + const port = await freePort(), output = []; + const child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true, + env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0', + SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '', + GREATEST_COVER_WARM: '0' } }); + child.stdout.on('data', d => output.push(String(d))); + child.stderr.on('data', d => output.push(String(d))); + t.after(async () => { + await new Promise(resolve => { if (child.exitCode !== null) return resolve(); child.once('exit', resolve); child.kill(); }); + fs.rmSync(root, { recursive: true, force: true }); + }); + const base = `http://127.0.0.1:${port}`; + let ready = false; + for (let n = 0; n < 100 && !ready; n++) { + ready = await fetch(base + '/health').then(r => r.ok).catch(() => false); + if (!ready) await sleep(50); + } + assert.ok(ready, output.join('')); + const get = (url, cookie = `sid=${token}`) => fetch(base + '/' + url, { headers: cookie ? { cookie } : {}, redirect: 'manual' }); + + const listing = await (await get('api/greatest')).json(); + const coverOf = id => listing.entries.find(entry => entry.id === id).cover; + assert.equal(coverOf(cached.id), `api/greatest-cover?id=${encodeURIComponent(cached.id)}&v=${version}`); + assert.match(coverOf(missing.id), /^api\/greatest-cover\?id=.+&v=[0-9a-f]{16}$/); + + const hit = await get(coverOf(cached.id)); + assert.equal(hit.status, 200); + assert.equal(hit.headers.get('content-type'), 'image/png'); + assert.equal(hit.headers.get('cache-control'), 'private, max-age=31536000, immutable'); + assert.deepEqual(Buffer.from(await hit.arrayBuffer()), PNG); + + const miss = await get(coverOf(missing.id)); + assert.equal(miss.status, 302); + assert.equal(miss.headers.get('location'), missing.cover); + assert.equal(miss.headers.get('cache-control'), 'no-store'); + + assert.equal((await get(coverOf(cached.id), null)).status, 401); + assert.equal((await get('api/greatest-cover?id=not-in-the-catalog')).status, 404); + assert.equal((await get('api/greatest-cover?id=../../db.json')).status, 404); +}); diff --git a/test/greatest-covers.test.js b/test/greatest-covers.test.js new file mode 100644 index 0000000..2e8417e --- /dev/null +++ b/test/greatest-covers.test.js @@ -0,0 +1,84 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { GreatestCovers } = require('../greatest/covers'); + +const PNG = Buffer.from('89504e470d0a1a0a0000000d49484452', 'hex'); +const image = (body = PNG, type = 'image/png') => new Response(body, { status: 200, headers: { 'content-type': type } }); + +function setup(t, responses, options = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-greatest-covers-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const calls = []; + const fetchImpl = async (url, init) => { + calls.push({ url, init }); + const next = responses[url]; + if (next instanceof Error) throw next; + return typeof next === 'function' ? next() : next || new Response('missing', { status: 404 }); + }; + const sleeps = []; + const covers = new GreatestCovers(dir, { fetchImpl, sleep: async ms => { sleeps.push(ms); }, ...options }); + return { dir, calls, sleeps, covers }; +} + +test('warm downloads each cover once and serves it from disk afterwards', async t => { + const a = 'https://img.example/a.png', b = 'https://img.example/b.jpg'; + const { dir, calls, sleeps, covers } = setup(t, { [a]: () => image(), [b]: () => image(PNG, 'image/jpeg') }); + const result = await covers.warm([a, b, a]); + assert.deepEqual(result, { fetched: 2, skipped: 1, failed: 0, stopped: null }); + assert.equal(calls.length, 2); + assert.deepEqual(sleeps, [1000], 'one pause between two downloads, none before the first'); + assert.ok(covers.file(a).endsWith('.png') && fs.existsSync(covers.file(a))); + assert.ok(covers.file(b).endsWith('.jpg')); + assert.equal(covers.file('https://img.example/other.png'), null); + + // A restarted reader finds the stored files and asks nobody. + const reopened = new GreatestCovers(dir, { fetchImpl: async () => { throw new Error('must not fetch'); } }); + assert.deepEqual(await reopened.warm([a, b]), { fetched: 0, skipped: 2, failed: 0, stopped: null }); + assert.equal(reopened.file(a), covers.file(a)); +}); + +test('a changed cover URL is a new file', async t => { + const { covers } = setup(t, { 'https://img.example/v1.png': () => image(), 'https://img.example/v2.png': () => image() }); + await covers.warm(['https://img.example/v1.png', 'https://img.example/v2.png']); + assert.notEqual(covers.file('https://img.example/v1.png'), covers.file('https://img.example/v2.png')); + assert.notEqual(covers.version('https://img.example/v1.png'), covers.version('https://img.example/v2.png')); +}); + +test('non-images, oversized files, plain http and errors are skipped', async t => { + const html = 'https://img.example/page', big = 'https://img.example/big.png', declaredBig = 'https://img.example/declared.png'; + const broken = 'https://img.example/broken.png', gone = 'https://img.example/gone.png'; + const { covers, calls } = setup(t, { + [html]: () => new Response('', { status: 200, headers: { 'content-type': 'text/html' } }), + [big]: () => image(Buffer.alloc(64)), + [declaredBig]: () => new Response(PNG, { status: 200, headers: { 'content-type': 'image/png', 'content-length': '999999' } }), + [broken]: new TypeError('fetch failed'), + }, { maxBytes: 32 }); + const result = await covers.warm([html, big, declaredBig, broken, gone, 'http://img.example/plain.png', '', null]); + assert.deepEqual(result, { fetched: 0, skipped: 0, failed: 5, stopped: null }); + assert.equal(calls.length, 5, 'plain http and empty URLs are never requested'); + for (const url of [html, big, declaredBig, broken, gone]) assert.equal(covers.file(url), null); + assert.deepEqual(fs.readdirSync(covers.dir).filter(name => !name.startsWith('.')), []); +}); + +test('429 and 403 stop the run at once', async t => { + for (const status of [429, 403]) { + const first = 'https://img.example/1.png', second = 'https://img.example/2.png'; + const { covers, calls } = setup(t, { [first]: () => new Response('', { status }), [second]: () => image() }); + const result = await covers.warm([first, second]); + assert.equal(result.stopped, `HTTP ${status}`); + assert.equal(calls.length, 1); + assert.equal(covers.file(second), null); + } +}); + +test('requests carry a timeout and a fixed user agent', async t => { + const a = 'https://img.example/a.png'; + const { covers, calls } = setup(t, { [a]: () => image() }); + await covers.warm([a]); + assert.ok(calls[0].init.signal instanceof AbortSignal); + assert.match(calls[0].init.headers['user-agent'], /^MangaCircus/); +}); diff --git a/test/library-index.test.js b/test/library-index.test.js index 64cec10..b3aca99 100644 --- a/test/library-index.test.js +++ b/test/library-index.test.js @@ -78,7 +78,7 @@ test('reader serves a saved index immediately after restart and verifies it in b const output = []; child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true, env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0', - SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '' } }); + SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', GREATEST_COVER_WARM: '0' } }); child.stdout.on('data', data => output.push(String(data))); child.stderr.on('data', data => output.push(String(data))); const base = `http://127.0.0.1:${port}`; diff --git a/test/services.test.js b/test/services.test.js index 448ea82..970be98 100644 --- a/test/services.test.js +++ b/test/services.test.js @@ -29,7 +29,7 @@ test('three services integrate collections, auth, custom covers and private heal const token = 'integration-test-only'; const env = { ...process.env, DATA: data, LIBRARY: library, SCRAPER_DATA: path.join(data, 'scraper'), PORT: String(rp), SCRAPER_PORT: String(sp), SCRAPER_HOST: '127.0.0.1', - READER_URL: base, SCRAPER_URL: scraper, SCRAPER_TOKEN: token, SCRAPER_TOKEN_FILE: '', SCAN_INTERVAL: '0', COOKIE_SECURE: '0' }; + READER_URL: base, SCRAPER_URL: scraper, SCRAPER_TOKEN: token, SCRAPER_TOKEN_FILE: '', SCAN_INTERVAL: '0', COOKIE_SECURE: '0', GREATEST_COVER_WARM: '0' }; for (const script of ['server.js', 'scraper/daemon.js']) { const child = spawn(process.execPath, [script], { cwd: root, env, windowsHide: true }); children.push(child); for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => output.push(chunk.toString())); diff --git a/tools/test-compose.js b/tools/test-compose.js index 79ab688..8fed0ae 100644 --- a/tools/test-compose.js +++ b/tools/test-compose.js @@ -69,6 +69,7 @@ for (const [index, name] of ['Moon Garden', 'Before the Rain', 'Iron Harbor'].en const env = { ...process.env, LIBRARY_PATH: library, DATA_PATH: data, SCRAPER_DATA_PATH: path.join(data, 'scraper'), SCRAPER_TOKEN_PATH: tokenFile, BACKUP_PATH: backupDir, READER_PORT: port, READER_BIND: '127.0.0.1', COOKIE_SECURE: '0', APP_IMAGE: process.env.APP_IMAGE || 'mangacircus:local', READER_IMAGE: process.env.APP_IMAGE || 'mangacircus:local', + GREATEST_COVER_WARM: '0', // QA stacks never download catalog covers ...(readingGateQa ? { READING_WINDOWS_JSON: '{"0":[],"1":[],"2":[],"3":[],"4":[],"5":[],"6":[]}', READING_TIMEZONE: 'Europe/Berlin', READING_QUIZ_UNLOCK_MINUTES: '60', READING_QUIZ_CATALOG_PATH: '/data/reading-quiz.json' } : { READING_WINDOWS_JSON: '', READING_QUIZ_CATALOG_PATH: '' }), diff --git a/tools/test-greatest-ui.js b/tools/test-greatest-ui.js index 8de5bc2..adad84d 100644 --- a/tools/test-greatest-ui.js +++ b/tools/test-greatest-ui.js @@ -40,7 +40,7 @@ async function freePort() { const scraperPort = fakeScraper.address().port; const child = spawn(process.execPath, ['server.js'], { cwd: root, windowsHide: true, env: { ...process.env, PORT: String(port), LIBRARY: library, DATA: data, SCAN_INTERVAL: '0', - FULL_SCAN_INTERVAL: '0', SCRAPER_URL: `http://127.0.0.1:${scraperPort}`, COOKIE_SECURE: '0' }, stdio: 'ignore' }); + FULL_SCAN_INTERVAL: '0', SCRAPER_URL: `http://127.0.0.1:${scraperPort}`, COOKIE_SECURE: '0', GREATEST_COVER_WARM: '0' }, stdio: 'ignore' }); let browser; try { for (let i = 0; i < 80; i++) {