Official release dates from MangaUpdates; peers lower ecosystem dates (BL-0048, BL-0049)
- scraper/metadata/release-dates.js fills x-dates.release from MangaUpdates publisher releases. MangaUpdates also lists English back-catalog re-releases, so a date is kept only inside the AniList original run, when no scanlation or source upload is more than 4 days earlier, and when it fits the release order (no lower chapter dated after a higher one). Seasons that restart chapter numbers are skipped; dates this job wrote and no longer accepts are withdrawn. - The scraper refreshes ongoing series nightly before its update checks, only while no download runs, and bumps the publish revision so the reader reconciles. - tools/release-dates.js for a whole library (dry run default). Production dry run: 728 chapters dated (Kagurabachi, JJK, Chainsaw Man, One Piece from 928). - Peer sync: chapters this library already has adopt a peer's earlier ecosystem date (job result ecosystemLowered); the peer listing now carries dates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e0dacd81c6
commit
7b52da2f9c
8 files changed
+402
-7
No files matched your search
+35
-3
@@ -12,7 +12,8 @@ const { publishManifest } = require('../scraper/lib/publisher');
|
||||
const { writeJsonAtomic } = require('../scraper/lib/json-file');
|
||||
const { sleep } = require('../scraper/lib/delay');
|
||||
const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity');
|
||||
const { fromPeer } = require('../chapter-dates');
|
||||
const { fromPeer, publishedDates } = require('../chapter-dates');
|
||||
const { parseKey, parseSpecialFolder } = require('../chapter-identity');
|
||||
|
||||
const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
|
||||
const store = () => new PeeringState(dataRoot());
|
||||
@@ -28,7 +29,37 @@ const infoFor = value => {
|
||||
};
|
||||
function refsOf(result) {
|
||||
return result.chapters.map(ref => ({ id: ref.id, number: ref.number, volume: ref.volume,
|
||||
book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount }));
|
||||
book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount,
|
||||
...(ref.dates && typeof ref.dates === 'object' ? { dates: ref.dates } : {}) }));
|
||||
}
|
||||
// Chapters this library already has adopt a peer's earlier ecosystem date (BL-0049): the
|
||||
// chapter entered Manga Circus there first. Later claims never raise a date.
|
||||
async function lowerEcosystemDates(mangaPath, refs, alias) {
|
||||
const claims = new Map();
|
||||
for (const ref of refs) {
|
||||
const claim = fromPeer(ref.dates, alias).ecosystem;
|
||||
const key = `${ref.volume}/${ref.number}`;
|
||||
if (claim && (!claims.has(key) || Date.parse(claim.at) < Date.parse(claims.get(key).at))) claims.set(key, claim);
|
||||
}
|
||||
let lowered = 0;
|
||||
if (!claims.size) return lowered;
|
||||
for (const book of await fs.readdir(mangaPath, { withFileTypes: true })) {
|
||||
const bookKey = book.isDirectory() && parseKey(book.name)?.key;
|
||||
if (!bookKey) continue;
|
||||
for (const chapter of await fs.readdir(path.join(mangaPath, book.name), { withFileTypes: true })) {
|
||||
const parsed = chapter.isDirectory() && (parseSpecialFolder(chapter.name) || parseKey(chapter.name));
|
||||
const claim = parsed && claims.get(`${bookKey}/${parsed.key}`);
|
||||
if (!claim) continue;
|
||||
const file = path.join(mangaPath, book.name, chapter.name, 'chapter.json');
|
||||
let meta;
|
||||
try { meta = JSON.parse(await fs.readFile(file, 'utf8')); } catch { continue; }
|
||||
const next = publishedDates(meta, { ecosystem: claim });
|
||||
if (next['x-dates'].ecosystem.at === meta['x-dates']?.ecosystem?.at) continue;
|
||||
await writeJsonAtomic(file, { ...meta, ...next });
|
||||
lowered++;
|
||||
}
|
||||
}
|
||||
return lowered;
|
||||
}
|
||||
function sameTitle(left, right) {
|
||||
return String(left || '').normalize('NFKC').trim().replace(/\s+/g, ' ').toLowerCase()
|
||||
@@ -102,6 +133,7 @@ async function run(info, item, options) {
|
||||
await writeJsonAtomic(path.join(manga.path, 'manga.json'), manga.meta);
|
||||
}
|
||||
const selected = selectByScope(catalog.chapters.filter(ref => shouldDownload(manga, ref, source, item.existingPolicy)), item.scope);
|
||||
const ecosystemLowered = manga ? await lowerEcosystemDates(manga.path, catalog.chapters, peer.alias) : 0;
|
||||
job.total = selected.length; job.jobCompleted = 0; job.current = null; await record();
|
||||
let publishedCount = 0;
|
||||
const identityToPersist = localIdentity || workIdentity;
|
||||
@@ -150,7 +182,7 @@ async function run(info, item, options) {
|
||||
if (index < selected.length - 1) { stop(); await sleep(Math.max(500, Number(options.settings.chapterMinMs || 500))); }
|
||||
}
|
||||
job.status = 'complete'; job.current = null; await record();
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length };
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length, ecosystemLowered };
|
||||
}
|
||||
function adapterForPeer(value) {
|
||||
const info = infoFor(value);
|
||||
|
||||
Reference in new issue
Block a user