Chapter dates: read date setting and tiered release dates (BL-0047)
- Chapter rows show when you first finished a chapter (setting: date + hour, date, off) from a new durable progress doneAt; older reads fall back to the reading clock's earliest remembered completion. - chapter.json keeps added (arrival in this library, now kept on replace) plus x-dates.source (source upload, bulk imports flagged), x-dates.release (official, for a later metadata job) and x-dates.ecosystem (first entry into any Manga Circus library; carried over peering, earliest wins, clamped). - Rows show one release date by tier: official > source upload > ecosystem > library, with all known dates in the tooltip (setting to hide). - Fixes: WeebCentral dates came from the chapter page, which has none (every chapter got the download time); they now come from the full chapter list. MangaDex no longer writes a group's publishAt into added. Peering sends dates. - tools/backfill-chapter-dates.js migrates an existing library (dry run default). - Coverage dry run on the server: docs/chapter-dates-coverage-2026-10-24.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b2bb34174c
commit
e0dacd81c6
21 files changed
+470
-32
No files matched your search
@@ -12,6 +12,7 @@ const { publishManifest } = require('../scraper/lib/publisher');
|
||||
const { writeJsonAtomic } = require('../scraper/lib/json-file');
|
||||
const { sleep } = require('../scraper/lib/delay');
|
||||
const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity');
|
||||
const { fromPeer } = require('../chapter-dates');
|
||||
|
||||
const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
|
||||
const store = () => new PeeringState(dataRoot());
|
||||
@@ -132,6 +133,8 @@ async function run(info, item, options) {
|
||||
}
|
||||
const publication = await publishManifest({ version: 1, series, chapters: [{
|
||||
number: ref.number, title: ref.title, book: { number: Number(ref.volume) || 1 },
|
||||
// The peer's ecosystem date travels along: the chapter entered Circus there, not here.
|
||||
dates: fromPeer(chapter.dates || ref.dates, peer.alias),
|
||||
source: { name: `Peer ${peer.alias} ${peer.id.slice(1, 7)}`, url: sourceUrl },
|
||||
pages: pages.map(page => ({ path: page.path })),
|
||||
}] }, { libraryRoot: options.libraryRoot, ...(manga ? { mangaDir: manga.path } : {}) });
|
||||
|
||||
+15
-3
@@ -9,6 +9,7 @@ const crypto = require('node:crypto');
|
||||
const { once } = require('node:events');
|
||||
const { PeeringState, fp, hash, requirePeer, grantStatus } = require('./state');
|
||||
const { parseKey, parseSpecialFolder } = require('../chapter-identity');
|
||||
const { peerDates } = require('../chapter-dates');
|
||||
const { resolveWorkIdentity } = require('../scraper/lib/work-identity');
|
||||
|
||||
const DATA = path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
|
||||
@@ -116,12 +117,23 @@ async function mangaIndex(peer, share) {
|
||||
if (!pages.length) continue;
|
||||
const id = `c${opaque(`${share.id}/${bookName}/${chapterName}`)}`;
|
||||
const chapterMeta = await jsonFile(path.join(chapterDir, 'chapter.json'));
|
||||
const dirStat = await fsp.stat(chapterDir);
|
||||
const item = { id, number: parsed.key, volume: book.key, title: String(chapterMeta.title || parsed.title || '').slice(0, 200),
|
||||
pageCount: pages.length, chapterDir, pages };
|
||||
pageCount: pages.length, chapterDir, pages, dates: peerDates(chapterMeta, dirStat.mtimeMs) };
|
||||
chapters.push(item); byId.set(id, item);
|
||||
if (chapters.length > 10000) fail(413, 'Shared manga chapter limit exceeded');
|
||||
}
|
||||
}
|
||||
// Every version of a chapter number offers the earliest ecosystem date known for it.
|
||||
const earliest = new Map();
|
||||
for (const item of chapters) {
|
||||
const key = `${item.volume}/${item.number}`, at = item.dates.ecosystem;
|
||||
if (at && (!earliest.has(key) || Date.parse(at) < Date.parse(earliest.get(key)))) earliest.set(key, at);
|
||||
}
|
||||
for (const item of chapters) {
|
||||
const at = earliest.get(`${item.volume}/${item.number}`);
|
||||
if (at) item.dates = { ...item.dates, ecosystem: at };
|
||||
}
|
||||
const result = { at: Date.now(), mangaDir, meta, chapters, byId };
|
||||
indexCache.set(key, result);
|
||||
return result;
|
||||
@@ -147,7 +159,7 @@ async function chapterManifest(peer, share, chapterId) {
|
||||
size: stat.size, mtimeMs: stat.mtimeMs, at: Date.now() });
|
||||
if (pages.length > 500) fail(413, 'Chapter page limit exceeded');
|
||||
}
|
||||
return { id: chapter.id, number: chapter.number, volume: chapter.volume, title: chapter.title, pages };
|
||||
return { id: chapter.id, number: chapter.number, volume: chapter.volume, title: chapter.title, dates: chapter.dates, pages };
|
||||
}
|
||||
async function streamBlob(req, res, peer, id) {
|
||||
const item = blobCache.get(`${peer.id}:${id}`);
|
||||
@@ -248,7 +260,7 @@ const server = https.createServer({ key: identity.key, cert: identity.cert, requ
|
||||
return json(res, 200, { id: share.id, title: String(index.meta.title || share.mangaId).slice(0, 200),
|
||||
workIdentity: resolveWorkIdentity(index.meta, share.mangaId, identity.id),
|
||||
...publicMetadata(index.meta), coverAvailable: !!coverFile(index.mangaDir),
|
||||
chapters: index.chapters.slice(cursor, cursor + 100).map(({ id, number, volume, title, pageCount }) => ({ id, number, volume, title, pageCount })),
|
||||
chapters: index.chapters.slice(cursor, cursor + 100).map(({ id, number, volume, title, pageCount, dates }) => ({ id, number, volume, title, pageCount, dates })),
|
||||
nextCursor: cursor + 100 < index.chapters.length ? String(cursor + 100) : null });
|
||||
}
|
||||
const chapterMatch = /^\/peer\/v1\/manga\/(m[0-9a-f]{24})\/chapters\/(c[0-9a-f]{32})$/.exec(url.pathname);
|
||||
|
||||
@@ -68,7 +68,8 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t
|
||||
await fsp.mkdir(chapterDir, { recursive: true });
|
||||
await fsp.writeFile(path.join(mangaDir, 'manga.json'), JSON.stringify({ title: 'Sample Manga', direction: 'ltr' }));
|
||||
await fsp.writeFile(path.join(mangaDir, 'cover.png'), PNG);
|
||||
await fsp.writeFile(path.join(chapterDir, 'chapter.json'), JSON.stringify({ title: 'The first chapter', source: { name: 'Local' } }));
|
||||
await fsp.writeFile(path.join(chapterDir, 'chapter.json'), JSON.stringify({ title: 'The first chapter', source: { name: 'Local' },
|
||||
added: '2026-10-01T08:00:00.000Z', 'x-dates': { version: 1, source: { at: '2026-09-30T15:00:00.000Z', name: 'WeebCentral', batch: false } } }));
|
||||
await fsp.writeFile(path.join(chapterDir, '0001.png'), PNG);
|
||||
const ga = await gateway(dataA, libA, portA, path.join(root, 'usage-a.json'));
|
||||
let gb = await gateway(dataB, libB, portB, path.join(root, 'usage-b.json'));
|
||||
@@ -134,7 +135,13 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t
|
||||
assert.equal(result.publishedCount, 1);
|
||||
assert.deepEqual(JSON.parse(await fsp.readFile(path.join(libB, 'Sample Manga', 'manga.json'), 'utf8'))['x-work-identity'], remote.workIdentity,
|
||||
'The stable work identity is stored with the imported peer edition');
|
||||
assert.equal(fs.existsSync(path.join(libB, 'Sample Manga', 'v01', 'c0001 The first chapter [Peer ServerA ' + inB.id.slice(1, 7) + ']', '0001.png')), true);
|
||||
const receivedDir = path.join(libB, 'Sample Manga', 'v01', 'c0001 The first chapter [Peer ServerA ' + inB.id.slice(1, 7) + ']');
|
||||
assert.equal(fs.existsSync(path.join(receivedDir, '0001.png')), true);
|
||||
const received = JSON.parse(await fsp.readFile(path.join(receivedDir, 'chapter.json'), 'utf8'));
|
||||
assert.deepEqual(received['x-dates'].ecosystem, { at: '2026-10-01T08:00:00.000Z', via: 'peer ServerA' },
|
||||
'The chapter keeps the date it first entered Manga Circus on A, not its arrival on B');
|
||||
assert.equal(received['x-dates'].source.at, '2026-09-30T15:00:00.000Z');
|
||||
assert.notEqual(received.added, '2026-10-01T08:00:00.000Z', '"added" is the arrival in B\'s own library');
|
||||
const second = await run(info, { scope: { mode: 'missing' }, existingPolicy: 'skip-any' },
|
||||
{ libraryRoot: libB, settings: { chapterMinMs: 500 }, shouldStop: () => false });
|
||||
assert.equal(second.publishedCount, 0);
|
||||
|
||||
Reference in new issue
Block a user