Discovery cache with per-kind TTL and size cap (BL-0051)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

discovery/cache.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+
+const DAY = 86400000;
+const KINDS = ['title', 'recs', 'creator', 'tagTop', 'fresh'];
+const TTL_MS = { title: 30 * DAY, recs: 30 * DAY, creator: 30 * DAY, tagTop: 7 * DAY, fresh: 7 * DAY };
+
+function checkKind(kind) {
+  if (!KINDS.includes(kind)) throw new TypeError(`Unknown discovery cache kind: ${kind}`);
+}
+
+class DiscoveryCache {
+  constructor(dir, { maxFiles = 5000, now = Date.now } = {}) {
+    this.dir = path.resolve(dir);
+    this.maxFiles = maxFiles;
+    this.now = now;
+  }
+  // base64url keeps any key (slashes, dots, spaces) inside the kind folder.
+  fileFor(kind, key) {
+    checkKind(kind);
+    return path.join(this.dir, kind, `${Buffer.from(String(key)).toString('base64url')}.json`);
+  }
+  readFile(file, kind) {
+    let entry;
+    try { entry = JSON.parse(fs.readFileSync(file, 'utf8')); }
+    catch (error) {
+      if (error.code === 'ENOENT') return null;
+      entry = null;
+    }
+    const valid = entry && entry.schema === 1 && entry.kind === kind && typeof entry.fetchedAt === 'string'
+      && Number.isFinite(Date.parse(entry.fetchedAt)) && 'data' in entry;
+    if (valid) return entry;
+    // A bad file would otherwise be re-read on every lookup; drop it so the caller refetches.
+    try { fs.rmSync(file, { force: true }); } catch {}
+    return null;
+  }
+  get(kind, key) { return this.readFile(this.fileFor(kind, key), kind); }
+  isFresh(kind, key) {
+    const entry = this.get(kind, key);
+    return !!entry && this.now() - Date.parse(entry.fetchedAt) < TTL_MS[kind];
+  }
+  put(kind, key, data) {
+    const file = this.fileFor(kind, key);
+    const entry = { schema: 1, kind, key, fetchedAt: new Date(this.now()).toISOString(), data };
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    const temp = `${file}.${crypto.randomBytes(6).toString('hex')}.tmp`;
+    try {
+      fs.writeFileSync(temp, JSON.stringify(entry), { mode: 0o600 });
+      fs.renameSync(temp, file);
+    } finally { try { fs.rmSync(temp, { force: true }); } catch {} }
+    this.prune();
+    return entry;
+  }
+  files(kind) {
+    try {
+      return fs.readdirSync(path.join(this.dir, kind)).filter((name) => name.endsWith('.json'))
+        .map((name) => path.join(this.dir, kind, name));
+    } catch { return []; }
+  }
+  list(kind) {
+    checkKind(kind);
+    return this.files(kind).map((file) => this.readFile(file, kind)).filter(Boolean);
+  }
+  stats() {
+    let files = 0, bytes = 0;
+    for (const kind of KINDS) for (const file of this.files(kind)) {
+      try { bytes += fs.statSync(file).size; files++; } catch {}
+    }
+    return { files, bytes };
+  }
+  // Oldest entries go first so the cache never grows past maxFiles.
+  prune() {
+    const all = [];
+    for (const kind of KINDS) for (const file of this.files(kind)) {
+      const entry = this.readFile(file, kind);
+      if (entry) all.push({ file, at: Date.parse(entry.fetchedAt) });
+    }
+    if (all.length <= this.maxFiles) return;
+    all.sort((a, b) => a.at - b.at);
+    for (const { file } of all.slice(0, all.length - this.maxFiles)) {
+      try { fs.rmSync(file, { force: true }); } catch {}
+    }
+  }
+}
+
+module.exports = { KINDS, TTL_MS, DiscoveryCache };

test/discovery-cache.test.js

@@ -0,0 +1,74 @@
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { DiscoveryCache } = require('../discovery/cache');
+
+const DAY = 86400000;
+
+function setup(t, options = {}) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'circus-dcache-'));
+  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
+  const clock = { t: Date.UTC(2026, 0, 1) };
+  const cache = new DiscoveryCache(dir, { ...options, now: () => clock.t });
+  return { dir, clock, cache };
+}
+
+test('cache round-trips an entry', (t) => {
+  const { clock, cache } = setup(t);
+  cache.put('title', 42, { a: 1 });
+  const entry = cache.get('title', 42);
+  assert.deepEqual(entry.data, { a: 1 });
+  assert.equal(entry.fetchedAt, new Date(clock.t).toISOString());
+  assert.equal(entry.schema, 1);
+  assert.equal(entry.kind, 'title');
+});
+
+test('freshness follows TTL per kind', (t) => {
+  const { clock, cache } = setup(t);
+  cache.put('fresh', 'global', []);
+  cache.put('title', 1, {});
+  clock.t += 8 * DAY;
+  assert.equal(cache.isFresh('fresh', 'global'), false);
+  assert.equal(cache.isFresh('title', 1), true);
+  clock.t += 23 * DAY;
+  assert.equal(cache.isFresh('title', 1), false);
+  assert.notEqual(cache.get('title', 1), null);
+  assert.equal(cache.list('title').length, 1);
+});
+
+test('prune keeps the newest maxFiles', (t) => {
+  const { clock, cache } = setup(t, { maxFiles: 3 });
+  for (let i = 1; i <= 5; i++) { clock.t += 1000; cache.put('title', i, { i }); }
+  assert.equal(cache.stats().files, 3);
+  assert.equal(cache.get('title', 1), null);
+  assert.equal(cache.get('title', 2), null);
+  assert.notEqual(cache.get('title', 5), null);
+});
+
+test('corrupt entry is deleted', (t) => {
+  const { dir, cache } = setup(t);
+  cache.put('recs', 7, {});
+  const file = path.join(dir, 'recs', fs.readdirSync(path.join(dir, 'recs'))[0]);
+  fs.writeFileSync(file, '{broken');
+  assert.equal(cache.get('recs', 7), null);
+  assert.equal(fs.existsSync(file), false);
+});
+
+test('odd keys are file-safe', (t) => {
+  const { dir, cache } = setup(t);
+  cache.put('tagTop', 'genre:Slice of Life/../x', [1]);
+  const files = fs.readdirSync(path.join(dir, 'tagTop'));
+  assert.equal(files.length, 1);
+  assert.deepEqual(fs.readdirSync(dir), ['tagTop']);
+  assert.deepEqual(cache.get('tagTop', 'genre:Slice of Life/../x').data, [1]);
+});
+
+test('unknown kind throws', (t) => {
+  const { cache } = setup(t);
+  assert.throws(() => cache.put('nope', 1, {}), TypeError);
+  assert.throws(() => cache.get('nope', 1), TypeError);
+});