Files
MaashkinoozheandClaude Opus 5.5 cd6a30a76b Discovery final review fixes: lean publish, robust scoring, cheap prune (BL-0051)
- Cache prune counts directory entries first and only reads files when over
  the cap; tests for cross-kind prune, schema-mismatch and corrupt entries.
- Scoring tolerates malformed manga.json and cached records (non-list tags,
  object or null relations, non-object items); a tag without a rank no longer
  passes the top-tag filter; authors without an integer AniList id are dropped.
- discovery.json publishes only candidates some feed shows; examined/clipped
  still describe the pool. Tests that asserted the published count equals the
  pool size now check the pool via candidatePool() and the published set
  against the feeds.
- The post-harvest rescore failure test now breaks discovery.json right before
  the publish (via a scraper stand-in) instead of relying on tags: 5, and a
  following successful harvest clears the stale lastError.
- Consent text names everything sent to AniList; paused label mentions admin
  harvests; the UI test asserts the XSS seed instead of skipping it.
- Slot-mix test proves the fresh/outside reservations matter.
- Docs: privacy note, fresh fallback reason, per-period cap, dry-run wording,
  published-candidate rule; plan/spec hygiene and spec status.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 09:43:31 +01:00

94 lines
3.5 KiB
JavaScript

'use strict';
const fs = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const DAY = 86400000;
const KINDS = ['title', 'recs', 'creator', 'tagTop', 'fresh'];
const TTL_MS = { title: 30 * DAY, recs: 30 * DAY, creator: 30 * DAY, tagTop: 7 * DAY, fresh: 7 * DAY };
function checkKind(kind) {
if (!KINDS.includes(kind)) throw new TypeError(`Unknown discovery cache kind: ${kind}`);
}
class DiscoveryCache {
constructor(dir, { maxFiles = 5000, now = Date.now } = {}) {
this.dir = path.resolve(dir);
this.maxFiles = maxFiles;
this.now = now;
}
// base64url keeps any key (slashes, dots, spaces) inside the kind folder.
fileFor(kind, key) {
checkKind(kind);
return path.join(this.dir, kind, `${Buffer.from(String(key)).toString('base64url')}.json`);
}
readFile(file, kind) {
let entry;
try { entry = JSON.parse(fs.readFileSync(file, 'utf8')); }
catch (error) {
if (error.code === 'ENOENT') return null;
entry = null;
}
const valid = entry && entry.schema === 1 && entry.kind === kind && typeof entry.fetchedAt === 'string'
&& Number.isFinite(Date.parse(entry.fetchedAt)) && 'data' in entry;
if (valid) return entry;
// A bad file would otherwise be re-read on every lookup; drop it so the caller refetches.
try { fs.rmSync(file, { force: true }); } catch {}
return null;
}
get(kind, key) { return this.readFile(this.fileFor(kind, key), kind); }
isFresh(kind, key) {
const entry = this.get(kind, key);
return !!entry && this.now() - Date.parse(entry.fetchedAt) < TTL_MS[kind];
}
put(kind, key, data) {
const file = this.fileFor(kind, key);
const entry = { schema: 1, kind, key, fetchedAt: new Date(this.now()).toISOString(), data };
fs.mkdirSync(path.dirname(file), { recursive: true });
const temp = `${file}.${crypto.randomBytes(6).toString('hex')}.tmp`;
try {
fs.writeFileSync(temp, JSON.stringify(entry), { mode: 0o600 });
fs.renameSync(temp, file);
} finally { try { fs.rmSync(temp, { force: true }); } catch {} }
this.prune();
return entry;
}
files(kind) {
try {
return fs.readdirSync(path.join(this.dir, kind)).filter((name) => name.endsWith('.json'))
.map((name) => path.join(this.dir, kind, name));
} catch { return []; }
}
list(kind) {
checkKind(kind);
return this.files(kind).map((file) => this.readFile(file, kind)).filter(Boolean);
}
stats() {
let files = 0, bytes = 0;
for (const kind of KINDS) for (const file of this.files(kind)) {
try { bytes += fs.statSync(file).size; files++; } catch {}
}
return { files, bytes };
}
// Oldest entries go first so the cache never grows past maxFiles. Every put() calls this, so
// under the cap it only counts directory entries; files are read only when something must go.
prune() {
const byKind = KINDS.map(kind => [kind, this.files(kind)]);
if (byKind.reduce((sum, [, files]) => sum + files.length, 0) <= this.maxFiles) return;
const all = [];
for (const [kind, files] of byKind) for (const file of files) {
// readFile drops corrupt and schema-mismatched files itself.
const entry = this.readFile(file, kind);
if (entry) all.push({ file, at: Date.parse(entry.fetchedAt) });
}
if (all.length <= this.maxFiles) return;
all.sort((a, b) => a.at - b.at);
for (const { file } of all.slice(0, all.length - this.maxFiles)) {
try { fs.rmSync(file, { force: true }); } catch {}
}
}
}
module.exports = { KINDS, TTL_MS, DiscoveryCache };