Official release dates from MangaUpdates; peers lower ecosystem dates (BL-0048, BL-0049)
- scraper/metadata/release-dates.js fills x-dates.release from MangaUpdates publisher releases. MangaUpdates also lists English back-catalog re-releases, so a date is kept only inside the AniList original run, when no scanlation or source upload is more than 4 days earlier, and when it fits the release order (no lower chapter dated after a higher one). Seasons that restart chapter numbers are skipped; dates this job wrote and no longer accepts are withdrawn. - The scraper refreshes ongoing series nightly before its update checks, only while no download runs, and bumps the publish revision so the reader reconciles. - tools/release-dates.js for a whole library (dry run default). Production dry run: 728 chapters dated (Kagurabachi, JJK, Chainsaw Man, One Piece from 928). - Peer sync: chapters this library already has adopt a peer's earlier ecosystem date (job result ecosystemLowered); the peer listing now carries dates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e0dacd81c6
commit
7b52da2f9c
8 files changed
+402
-7
No files matched your search
+35
-3
@@ -12,7 +12,8 @@ const { publishManifest } = require('../scraper/lib/publisher');
|
||||
const { writeJsonAtomic } = require('../scraper/lib/json-file');
|
||||
const { sleep } = require('../scraper/lib/delay');
|
||||
const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity');
|
||||
const { fromPeer } = require('../chapter-dates');
|
||||
const { fromPeer, publishedDates } = require('../chapter-dates');
|
||||
const { parseKey, parseSpecialFolder } = require('../chapter-identity');
|
||||
|
||||
const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
|
||||
const store = () => new PeeringState(dataRoot());
|
||||
@@ -28,7 +29,37 @@ const infoFor = value => {
|
||||
};
|
||||
function refsOf(result) {
|
||||
return result.chapters.map(ref => ({ id: ref.id, number: ref.number, volume: ref.volume,
|
||||
book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount }));
|
||||
book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount,
|
||||
...(ref.dates && typeof ref.dates === 'object' ? { dates: ref.dates } : {}) }));
|
||||
}
|
||||
// Chapters this library already has adopt a peer's earlier ecosystem date (BL-0049): the
|
||||
// chapter entered Manga Circus there first. Later claims never raise a date.
|
||||
async function lowerEcosystemDates(mangaPath, refs, alias) {
|
||||
const claims = new Map();
|
||||
for (const ref of refs) {
|
||||
const claim = fromPeer(ref.dates, alias).ecosystem;
|
||||
const key = `${ref.volume}/${ref.number}`;
|
||||
if (claim && (!claims.has(key) || Date.parse(claim.at) < Date.parse(claims.get(key).at))) claims.set(key, claim);
|
||||
}
|
||||
let lowered = 0;
|
||||
if (!claims.size) return lowered;
|
||||
for (const book of await fs.readdir(mangaPath, { withFileTypes: true })) {
|
||||
const bookKey = book.isDirectory() && parseKey(book.name)?.key;
|
||||
if (!bookKey) continue;
|
||||
for (const chapter of await fs.readdir(path.join(mangaPath, book.name), { withFileTypes: true })) {
|
||||
const parsed = chapter.isDirectory() && (parseSpecialFolder(chapter.name) || parseKey(chapter.name));
|
||||
const claim = parsed && claims.get(`${bookKey}/${parsed.key}`);
|
||||
if (!claim) continue;
|
||||
const file = path.join(mangaPath, book.name, chapter.name, 'chapter.json');
|
||||
let meta;
|
||||
try { meta = JSON.parse(await fs.readFile(file, 'utf8')); } catch { continue; }
|
||||
const next = publishedDates(meta, { ecosystem: claim });
|
||||
if (next['x-dates'].ecosystem.at === meta['x-dates']?.ecosystem?.at) continue;
|
||||
await writeJsonAtomic(file, { ...meta, ...next });
|
||||
lowered++;
|
||||
}
|
||||
}
|
||||
return lowered;
|
||||
}
|
||||
function sameTitle(left, right) {
|
||||
return String(left || '').normalize('NFKC').trim().replace(/\s+/g, ' ').toLowerCase()
|
||||
@@ -102,6 +133,7 @@ async function run(info, item, options) {
|
||||
await writeJsonAtomic(path.join(manga.path, 'manga.json'), manga.meta);
|
||||
}
|
||||
const selected = selectByScope(catalog.chapters.filter(ref => shouldDownload(manga, ref, source, item.existingPolicy)), item.scope);
|
||||
const ecosystemLowered = manga ? await lowerEcosystemDates(manga.path, catalog.chapters, peer.alias) : 0;
|
||||
job.total = selected.length; job.jobCompleted = 0; job.current = null; await record();
|
||||
let publishedCount = 0;
|
||||
const identityToPersist = localIdentity || workIdentity;
|
||||
@@ -150,7 +182,7 @@ async function run(info, item, options) {
|
||||
if (index < selected.length - 1) { stop(); await sleep(Math.max(500, Number(options.settings.chapterMinMs || 500))); }
|
||||
}
|
||||
job.status = 'complete'; job.current = null; await record();
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length };
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length, ecosystemLowered };
|
||||
}
|
||||
function adapterForPeer(value) {
|
||||
const info = infoFor(value);
|
||||
|
||||
@@ -142,9 +142,15 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t
|
||||
'The chapter keeps the date it first entered Manga Circus on A, not its arrival on B');
|
||||
assert.equal(received['x-dates'].source.at, '2026-09-30T15:00:00.000Z');
|
||||
assert.notEqual(received.added, '2026-10-01T08:00:00.000Z', '"added" is the arrival in B\'s own library');
|
||||
// B claims it got the chapter later on its own: the next sync lowers it to A's date again.
|
||||
await fsp.writeFile(path.join(receivedDir, 'chapter.json'), JSON.stringify({ ...received,
|
||||
'x-dates': { ...received['x-dates'], ecosystem: { at: '2026-10-20T00:00:00.000Z', via: 'this server' } } }));
|
||||
const second = await run(info, { scope: { mode: 'missing' }, existingPolicy: 'skip-any' },
|
||||
{ libraryRoot: libB, settings: { chapterMinMs: 500 }, shouldStop: () => false });
|
||||
assert.equal(second.publishedCount, 0);
|
||||
assert.equal(second.ecosystemLowered, 1, 'An existing chapter adopts the peer\'s earlier ecosystem date');
|
||||
assert.deepEqual(JSON.parse(await fsp.readFile(path.join(receivedDir, 'chapter.json'), 'utf8'))['x-dates'].ecosystem,
|
||||
{ at: '2026-10-01T08:00:00.000Z', via: 'peer ServerA' });
|
||||
|
||||
// Simulate B adding one chapter to its imported edition, then sharing it back.
|
||||
// A must identify its original folder by the carried origin identity and fetch
|
||||
|
||||
Reference in new issue
Block a user