Chapter dates: read date setting and tiered release dates (BL-0047)

- Chapter rows show when you first finished a chapter (setting: date + hour,
  date, off) from a new durable progress doneAt; older reads fall back to the
  reading clock's earliest remembered completion.
- chapter.json keeps added (arrival in this library, now kept on replace) plus
  x-dates.source (source upload, bulk imports flagged), x-dates.release
  (official, for a later metadata job) and x-dates.ecosystem (first entry into
  any Manga Circus library; carried over peering, earliest wins, clamped).
- Rows show one release date by tier: official > source upload > ecosystem >
  library, with all known dates in the tooltip (setting to hide).
- Fixes: WeebCentral dates came from the chapter page, which has none (every
  chapter got the download time); they now come from the full chapter list.
  MangaDex no longer writes a group's publishAt into added. Peering sends dates.
- tools/backfill-chapter-dates.js migrates an existing library (dry run default).
- Coverage dry run on the server: docs/chapter-dates-coverage-2026-10-24.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
MaashkinoozheandClaude Opus 5.5 committed 2026-10-24 01:01:33 +02:00
1 parent b2bb34174c
commit e0dacd81c6
21 files changed
+470 -32

No files matched your search

+3
View File
@@ -12,6 +12,7 @@ const { publishManifest } = require('../scraper/lib/publisher');
const { writeJsonAtomic } = require('../scraper/lib/json-file');
const { sleep } = require('../scraper/lib/delay');
const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity');
const { fromPeer } = require('../chapter-dates');
const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
const store = () => new PeeringState(dataRoot());
@@ -132,6 +133,8 @@ async function run(info, item, options) {
}
const publication = await publishManifest({ version: 1, series, chapters: [{
number: ref.number, title: ref.title, book: { number: Number(ref.volume) || 1 },
// The peer's ecosystem date travels along: the chapter entered Circus there, not here.
dates: fromPeer(chapter.dates || ref.dates, peer.alias),
source: { name: `Peer ${peer.alias} ${peer.id.slice(1, 7)}`, url: sourceUrl },
pages: pages.map(page => ({ path: page.path })),
}] }, { libraryRoot: options.libraryRoot, ...(manga ? { mangaDir: manga.path } : {}) });
+15 -3
View File
@@ -9,6 +9,7 @@ const crypto = require('node:crypto');
const { once } = require('node:events');
const { PeeringState, fp, hash, requirePeer, grantStatus } = require('./state');
const { parseKey, parseSpecialFolder } = require('../chapter-identity');
const { peerDates } = require('../chapter-dates');
const { resolveWorkIdentity } = require('../scraper/lib/work-identity');
const DATA = path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
@@ -116,12 +117,23 @@ async function mangaIndex(peer, share) {
if (!pages.length) continue;
const id = `c${opaque(`${share.id}/${bookName}/${chapterName}`)}`;
const chapterMeta = await jsonFile(path.join(chapterDir, 'chapter.json'));
const dirStat = await fsp.stat(chapterDir);
const item = { id, number: parsed.key, volume: book.key, title: String(chapterMeta.title || parsed.title || '').slice(0, 200),
pageCount: pages.length, chapterDir, pages };
pageCount: pages.length, chapterDir, pages, dates: peerDates(chapterMeta, dirStat.mtimeMs) };
chapters.push(item); byId.set(id, item);
if (chapters.length > 10000) fail(413, 'Shared manga chapter limit exceeded');
}
}
// Every version of a chapter number offers the earliest ecosystem date known for it.
const earliest = new Map();
for (const item of chapters) {
const key = `${item.volume}/${item.number}`, at = item.dates.ecosystem;
if (at && (!earliest.has(key) || Date.parse(at) < Date.parse(earliest.get(key)))) earliest.set(key, at);
}
for (const item of chapters) {
const at = earliest.get(`${item.volume}/${item.number}`);
if (at) item.dates = { ...item.dates, ecosystem: at };
}
const result = { at: Date.now(), mangaDir, meta, chapters, byId };
indexCache.set(key, result);
return result;
@@ -147,7 +159,7 @@ async function chapterManifest(peer, share, chapterId) {
size: stat.size, mtimeMs: stat.mtimeMs, at: Date.now() });
if (pages.length > 500) fail(413, 'Chapter page limit exceeded');
}
return { id: chapter.id, number: chapter.number, volume: chapter.volume, title: chapter.title, pages };
return { id: chapter.id, number: chapter.number, volume: chapter.volume, title: chapter.title, dates: chapter.dates, pages };
}
async function streamBlob(req, res, peer, id) {
const item = blobCache.get(`${peer.id}:${id}`);
@@ -248,7 +260,7 @@ const server = https.createServer({ key: identity.key, cert: identity.cert, requ
return json(res, 200, { id: share.id, title: String(index.meta.title || share.mangaId).slice(0, 200),
workIdentity: resolveWorkIdentity(index.meta, share.mangaId, identity.id),
...publicMetadata(index.meta), coverAvailable: !!coverFile(index.mangaDir),
chapters: index.chapters.slice(cursor, cursor + 100).map(({ id, number, volume, title, pageCount }) => ({ id, number, volume, title, pageCount })),
chapters: index.chapters.slice(cursor, cursor + 100).map(({ id, number, volume, title, pageCount, dates }) => ({ id, number, volume, title, pageCount, dates })),
nextCursor: cursor + 100 < index.chapters.length ? String(cursor + 100) : null });
}
const chapterMatch = /^\/peer\/v1\/manga\/(m[0-9a-f]{24})\/chapters\/(c[0-9a-f]{32})$/.exec(url.pathname);
+9 -2
View File
@@ -68,7 +68,8 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t
await fsp.mkdir(chapterDir, { recursive: true });
await fsp.writeFile(path.join(mangaDir, 'manga.json'), JSON.stringify({ title: 'Sample Manga', direction: 'ltr' }));
await fsp.writeFile(path.join(mangaDir, 'cover.png'), PNG);
await fsp.writeFile(path.join(chapterDir, 'chapter.json'), JSON.stringify({ title: 'The first chapter', source: { name: 'Local' } }));
await fsp.writeFile(path.join(chapterDir, 'chapter.json'), JSON.stringify({ title: 'The first chapter', source: { name: 'Local' },
added: '2026-10-01T08:00:00.000Z', 'x-dates': { version: 1, source: { at: '2026-09-30T15:00:00.000Z', name: 'WeebCentral', batch: false } } }));
await fsp.writeFile(path.join(chapterDir, '0001.png'), PNG);
const ga = await gateway(dataA, libA, portA, path.join(root, 'usage-a.json'));
let gb = await gateway(dataB, libB, portB, path.join(root, 'usage-b.json'));
@@ -134,7 +135,13 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t
assert.equal(result.publishedCount, 1);
assert.deepEqual(JSON.parse(await fsp.readFile(path.join(libB, 'Sample Manga', 'manga.json'), 'utf8'))['x-work-identity'], remote.workIdentity,
'The stable work identity is stored with the imported peer edition');
assert.equal(fs.existsSync(path.join(libB, 'Sample Manga', 'v01', 'c0001 The first chapter [Peer ServerA ' + inB.id.slice(1, 7) + ']', '0001.png')), true);
const receivedDir = path.join(libB, 'Sample Manga', 'v01', 'c0001 The first chapter [Peer ServerA ' + inB.id.slice(1, 7) + ']');
assert.equal(fs.existsSync(path.join(receivedDir, '0001.png')), true);
const received = JSON.parse(await fsp.readFile(path.join(receivedDir, 'chapter.json'), 'utf8'));
assert.deepEqual(received['x-dates'].ecosystem, { at: '2026-10-01T08:00:00.000Z', via: 'peer ServerA' },
'The chapter keeps the date it first entered Manga Circus on A, not its arrival on B');
assert.equal(received['x-dates'].source.at, '2026-09-30T15:00:00.000Z');
assert.notEqual(received.added, '2026-10-01T08:00:00.000Z', '"added" is the arrival in B\'s own library');
const second = await run(info, { scope: { mode: 'missing' }, existingPolicy: 'skip-any' },
{ libraryRoot: libB, settings: { chapterMinMs: 500 }, shouldStop: () => false });
assert.equal(second.publishedCount, 0);