Official release dates from MangaUpdates; peers lower ecosystem dates (BL-0048, BL-0049)
- scraper/metadata/release-dates.js fills x-dates.release from MangaUpdates publisher releases. MangaUpdates also lists English back-catalog re-releases, so a date is kept only inside the AniList original run, when no scanlation or source upload is more than 4 days earlier, and when it fits the release order (no lower chapter dated after a higher one). Seasons that restart chapter numbers are skipped; dates this job wrote and no longer accepts are withdrawn. - The scraper refreshes ongoing series nightly before its update checks, only while no download runs, and bumps the publish revision so the reader reconciles. - tools/release-dates.js for a whole library (dry run default). Production dry run: 728 chapters dated (Kagurabachi, JJK, Chainsaw Man, One Piece from 928). - Peer sync: chapters this library already has adopt a peer's earlier ecosystem date (job result ecosystemLowered); the peer listing now carries dates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e0dacd81c6
commit
7b52da2f9c
8 files changed
+402
-7
No files matched your search
+16
-2
@@ -229,8 +229,22 @@ An AniList refresh with a different ID is stopped for review before chapter publ
|
||||
- `source`: when the source site uploaded this version (WeebCentral's full chapter list,
|
||||
MangaDex `publishAt`). `batch: true` means it arrived with 3 or more other uploads
|
||||
less than 10 minutes apart, which is a site import, not a release.
|
||||
- `release`: the official public release (`precision` `day`, `month` or `year`). Not
|
||||
filled yet; a metadata job will add it.
|
||||
- `release`: the official public release (`precision` `day`, `month` or `year`). It is
|
||||
filled from MangaUpdates releases by publishers such as Viz, MANGA Plus or Webtoon
|
||||
(`scraper/metadata/release-dates.js`). Those lists include English re-releases of old
|
||||
chapters, so a date is kept only when all of these hold:
|
||||
- it lies within the work's original run on AniList;
|
||||
- no scanlation or source upload shows the chapter more than 4 days earlier;
|
||||
- it fits the release order: walking down from the newest chapter, a lower chapter
|
||||
can't be dated more than 2 days after a higher one.
|
||||
|
||||
Series whose chapter numbers restart in a later book are skipped.
|
||||
- The scraper refreshes ongoing series nightly before its update checks, but only
|
||||
while no download is running.
|
||||
- `node tools/release-dates.js <library>` covers a whole library: a dry run by
|
||||
default, then `--apply --writers-stopped`.
|
||||
- A date that this job wrote and no longer accepts is removed. Release dates from any
|
||||
other source stay.
|
||||
- `ecosystem`: when the chapter first entered any Manga Circus library. Peer transfers
|
||||
carry it, and the earliest known date always wins. A peer's claim can't be earlier than
|
||||
the source upload or later than now.
|
||||
|
||||
+4
-1
@@ -118,7 +118,10 @@ date at which any version of that chapter number entered a Manga Circus library
|
||||
to the sender. Older chapters fall back to their `added` date. The receiver stores the
|
||||
date as `x-dates.ecosystem` (`via: "peer <alias>"`) and only lets it lower an existing
|
||||
date, never raise it. See `docs/library-format.md`. Older peers omit `dates`, and the
|
||||
receiver then uses its own arrival time.
|
||||
receiver then uses its own arrival time. A sync also lowers the ecosystem date of
|
||||
chapters this library already has, including ones it scraped itself, when the peer
|
||||
knows an earlier date for the same book and chapter number. The job reports these as
|
||||
`ecosystemLowered`. The reader shows the new date after its next full reconciliation.
|
||||
| `GET /peer/v1/blobs/{id}` | One image, optional `Range: bytes=N-` and `If-Match`; 206 on resume |
|
||||
|
||||
Unknown/unshared IDs return 404. Revoked serving returns 403; bad/missing grants
|
||||
|
||||
+35
-3
@@ -12,7 +12,8 @@ const { publishManifest } = require('../scraper/lib/publisher');
|
||||
const { writeJsonAtomic } = require('../scraper/lib/json-file');
|
||||
const { sleep } = require('../scraper/lib/delay');
|
||||
const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity');
|
||||
const { fromPeer } = require('../chapter-dates');
|
||||
const { fromPeer, publishedDates } = require('../chapter-dates');
|
||||
const { parseKey, parseSpecialFolder } = require('../chapter-identity');
|
||||
|
||||
const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
|
||||
const store = () => new PeeringState(dataRoot());
|
||||
@@ -28,7 +29,37 @@ const infoFor = value => {
|
||||
};
|
||||
function refsOf(result) {
|
||||
return result.chapters.map(ref => ({ id: ref.id, number: ref.number, volume: ref.volume,
|
||||
book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount }));
|
||||
book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount,
|
||||
...(ref.dates && typeof ref.dates === 'object' ? { dates: ref.dates } : {}) }));
|
||||
}
|
||||
// Chapters this library already has adopt a peer's earlier ecosystem date (BL-0049): the
|
||||
// chapter entered Manga Circus there first. Later claims never raise a date.
|
||||
async function lowerEcosystemDates(mangaPath, refs, alias) {
|
||||
const claims = new Map();
|
||||
for (const ref of refs) {
|
||||
const claim = fromPeer(ref.dates, alias).ecosystem;
|
||||
const key = `${ref.volume}/${ref.number}`;
|
||||
if (claim && (!claims.has(key) || Date.parse(claim.at) < Date.parse(claims.get(key).at))) claims.set(key, claim);
|
||||
}
|
||||
let lowered = 0;
|
||||
if (!claims.size) return lowered;
|
||||
for (const book of await fs.readdir(mangaPath, { withFileTypes: true })) {
|
||||
const bookKey = book.isDirectory() && parseKey(book.name)?.key;
|
||||
if (!bookKey) continue;
|
||||
for (const chapter of await fs.readdir(path.join(mangaPath, book.name), { withFileTypes: true })) {
|
||||
const parsed = chapter.isDirectory() && (parseSpecialFolder(chapter.name) || parseKey(chapter.name));
|
||||
const claim = parsed && claims.get(`${bookKey}/${parsed.key}`);
|
||||
if (!claim) continue;
|
||||
const file = path.join(mangaPath, book.name, chapter.name, 'chapter.json');
|
||||
let meta;
|
||||
try { meta = JSON.parse(await fs.readFile(file, 'utf8')); } catch { continue; }
|
||||
const next = publishedDates(meta, { ecosystem: claim });
|
||||
if (next['x-dates'].ecosystem.at === meta['x-dates']?.ecosystem?.at) continue;
|
||||
await writeJsonAtomic(file, { ...meta, ...next });
|
||||
lowered++;
|
||||
}
|
||||
}
|
||||
return lowered;
|
||||
}
|
||||
function sameTitle(left, right) {
|
||||
return String(left || '').normalize('NFKC').trim().replace(/\s+/g, ' ').toLowerCase()
|
||||
@@ -102,6 +133,7 @@ async function run(info, item, options) {
|
||||
await writeJsonAtomic(path.join(manga.path, 'manga.json'), manga.meta);
|
||||
}
|
||||
const selected = selectByScope(catalog.chapters.filter(ref => shouldDownload(manga, ref, source, item.existingPolicy)), item.scope);
|
||||
const ecosystemLowered = manga ? await lowerEcosystemDates(manga.path, catalog.chapters, peer.alias) : 0;
|
||||
job.total = selected.length; job.jobCompleted = 0; job.current = null; await record();
|
||||
let publishedCount = 0;
|
||||
const identityToPersist = localIdentity || workIdentity;
|
||||
@@ -150,7 +182,7 @@ async function run(info, item, options) {
|
||||
if (index < selected.length - 1) { stop(); await sleep(Math.max(500, Number(options.settings.chapterMinMs || 500))); }
|
||||
}
|
||||
job.status = 'complete'; job.current = null; await record();
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length };
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length, ecosystemLowered };
|
||||
}
|
||||
function adapterForPeer(value) {
|
||||
const info = infoFor(value);
|
||||
|
||||
@@ -142,9 +142,15 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t
|
||||
'The chapter keeps the date it first entered Manga Circus on A, not its arrival on B');
|
||||
assert.equal(received['x-dates'].source.at, '2026-09-30T15:00:00.000Z');
|
||||
assert.notEqual(received.added, '2026-10-01T08:00:00.000Z', '"added" is the arrival in B\'s own library');
|
||||
// B claims it got the chapter later on its own: the next sync lowers it to A's date again.
|
||||
await fsp.writeFile(path.join(receivedDir, 'chapter.json'), JSON.stringify({ ...received,
|
||||
'x-dates': { ...received['x-dates'], ecosystem: { at: '2026-10-20T00:00:00.000Z', via: 'this server' } } }));
|
||||
const second = await run(info, { scope: { mode: 'missing' }, existingPolicy: 'skip-any' },
|
||||
{ libraryRoot: libB, settings: { chapterMinMs: 500 }, shouldStop: () => false });
|
||||
assert.equal(second.publishedCount, 0);
|
||||
assert.equal(second.ecosystemLowered, 1, 'An existing chapter adopts the peer\'s earlier ecosystem date');
|
||||
assert.deepEqual(JSON.parse(await fsp.readFile(path.join(receivedDir, 'chapter.json'), 'utf8'))['x-dates'].ecosystem,
|
||||
{ at: '2026-10-01T08:00:00.000Z', via: 'peer ServerA' });
|
||||
|
||||
// Simulate B adding one chapter to its imported edition, then sharing it back.
|
||||
// A must identify its original folder by the carried origin identity and fetch
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
'use strict';
|
||||
// Official chapter release dates (BL-0048) from MangaUpdates publisher releases.
|
||||
// MangaUpdates also lists English digital re-releases of old chapters as "official"
|
||||
// (Naruto ch. 700 shows 2020; it came out in Japan in 2014). A date is accepted only when
|
||||
// 1. it lies inside the work's original run (AniList start - 7 days ... end + 30 days), and
|
||||
// 2. nothing shows the chapter earlier: no scanlation release in MangaUpdates and no
|
||||
// source upload (bulk or not) more than EARLY_TOLERANCE_DAYS before it, and
|
||||
// 3. it fits the release order (see officialDates).
|
||||
// Simulpub releases pass all three; back-catalog re-releases fail at least one.
|
||||
const fs = require('node:fs').promises;
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
const { parseKey } = require('../../chapter-identity');
|
||||
const { writeJsonAtomic } = require('../lib/json-file');
|
||||
|
||||
const DAY = 864e5;
|
||||
const EARLY_TOLERANCE_DAYS = 4; // early scans and source uploads of weekly series arrive 1-3 days before the official date
|
||||
const ORDER_TOLERANCE_DAYS = 2; // double releases and day-granular dates may put a lower chapter a day or two later
|
||||
const OFFICIAL = new Set(['viz', 'viz media', 'manga plus', 'mangaplus', 'shueisha', 'kodansha', 'kodansha usa', 'k manga',
|
||||
'yen press', 'seven seas', 'square enix', 'square enix manga', 'manga up!', 'comikey', 'azuki', 'tappytoon', 'webtoon',
|
||||
'line webtoon', 'webtoons', 'tapas', 'pocket comics', 'manta', 'lezhin', 'lezhin comics', 'j-novel club', 'dark horse',
|
||||
'vertical', 'denpa', 'tokyopop', 'ize press', 'one peace books', 'inky pen', 'kakaopage', 'kakao webtoon', 'naver webtoon']);
|
||||
const UA = 'MangaCircus/1.0 (self-hosted manga reader; chapter release dates)';
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const norm = value => String(value || '').toLowerCase().normalize('NFKD').replace(/[^a-z0-9]+/g, '');
|
||||
// "001", "1.50" and "1" are the same chapter; ranges, volumes and letters are not single chapters.
|
||||
const chapterKey = value => (/^\s*\d+(?:\.\d+)?\s*$/.test(String(value ?? '')) ? String(Number(value)) : null);
|
||||
const isOfficial = groups => groups.some(name => OFFICIAL.has(String(name).toLowerCase().trim()));
|
||||
|
||||
async function cached(cacheDir, key, maxAgeMs, load) {
|
||||
const file = cacheDir && path.join(cacheDir, crypto.createHash('sha256').update(key).digest('hex').slice(0, 32) + '.json');
|
||||
if (file) {
|
||||
try {
|
||||
const stat = await fs.stat(file);
|
||||
if (Date.now() - stat.mtimeMs < maxAgeMs) return JSON.parse(await fs.readFile(file, 'utf8'));
|
||||
} catch { /* not cached */ }
|
||||
}
|
||||
const value = await load();
|
||||
if (file) { await fs.mkdir(cacheDir, { recursive: true }); await writeJsonAtomic(file, value); }
|
||||
return value;
|
||||
}
|
||||
|
||||
class ReleaseDateSources {
|
||||
constructor(options = {}) {
|
||||
this.cacheDir = options.cacheDir || null;
|
||||
this.minGapMs = options.minGapMs ?? 1100;
|
||||
this.fetch = options.fetch || globalThis.fetch;
|
||||
this.last = 0;
|
||||
}
|
||||
async request(url, body) {
|
||||
const wait = this.last + this.minGapMs - Date.now();
|
||||
if (wait > 0) await sleep(wait);
|
||||
this.last = Date.now();
|
||||
const response = await this.fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', 'user-agent': UA },
|
||||
body: JSON.stringify(body), signal: AbortSignal.timeout(30000) });
|
||||
if (!response.ok) throw new Error(`${new URL(url).hostname} HTTP ${response.status}`);
|
||||
return response.json();
|
||||
}
|
||||
// MangaUpdates series whose matched name equals one of the manga's titles and whose year fits.
|
||||
async findSeries(meta) {
|
||||
const titles = [meta.title, ...(Array.isArray(meta.altTitles) ? meta.altTitles : [])].map(norm).filter(Boolean);
|
||||
const year = Number(meta.year) || null;
|
||||
const result = await cached(this.cacheDir, `mu-search:${meta.title}`, 30 * DAY,
|
||||
() => this.request('https://api.mangaupdates.com/v1/series/search', { search: String(meta.title || ''), perpage: 10 }));
|
||||
for (const hit of result.results || []) {
|
||||
const record = hit.record || {}, hitYear = Number(record.year) || null;
|
||||
if (year && hitYear && Math.abs(hitYear - year) > 1) continue;
|
||||
if ([record.title, hit.hit_title].map(norm).some(name => name && titles.includes(name))) {
|
||||
return { id: String(record.series_id), title: record.title, url: record.url || null };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
async releases(seriesId, { pages = Infinity, maxAgeMs = 7 * DAY } = {}) {
|
||||
return cached(this.cacheDir, `mu-releases:${seriesId}:${pages}`, maxAgeMs, async () => {
|
||||
const rows = [];
|
||||
for (let page = 1; page <= pages; page++) {
|
||||
const result = await this.request('https://api.mangaupdates.com/v1/releases/search',
|
||||
{ search: String(seriesId), search_type: 'series', perpage: 100, page, orderby: 'date' });
|
||||
for (const { record } of result.results || []) {
|
||||
if (record?.release_date) rows.push({ chapter: record.chapter, date: record.release_date, groups: (record.groups || []).map(g => g.name) });
|
||||
}
|
||||
if (!result.results?.length || page * 100 >= (result.total_hits || 0)) break;
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
async originalRun(anilistId) {
|
||||
if (!anilistId) return null;
|
||||
const result = await cached(this.cacheDir, `anilist-run:${anilistId}`, 30 * DAY, () => this.request('https://graphql.anilist.co', {
|
||||
query: 'query ($id: Int) { Media(id: $id, type: MANGA) { status startDate { year month day } endDate { year month day } } }',
|
||||
variables: { id: Number(anilistId) },
|
||||
}));
|
||||
const media = result.data?.Media;
|
||||
if (!media) return null;
|
||||
const date = (d, end) => (d?.year ? Date.UTC(d.year, (d.month || (end ? 12 : 1)) - 1, d.day || (end ? 28 : 1)) : null);
|
||||
return { status: media.status, start: date(media.startDate, false), end: media.status === 'RELEASING' ? null : date(media.endDate, true) };
|
||||
}
|
||||
}
|
||||
|
||||
// Pure decision: official dates per chapter key, with the reasons others were rejected.
|
||||
// evidence: Map chapterKey -> earliest other time (ms) the library knows for that chapter.
|
||||
function officialDates(releases, run, evidence = new Map()) {
|
||||
const official = new Map(), scans = new Map();
|
||||
for (const row of releases) {
|
||||
const key = chapterKey(row.chapter), time = Date.parse(row.date);
|
||||
if (!key || !Number.isFinite(time)) continue;
|
||||
const target = isOfficial(row.groups) ? official : scans;
|
||||
const prior = target.get(key);
|
||||
if (!prior || time < prior.time) target.set(key, { time, date: String(row.date).slice(0, 10), groups: row.groups });
|
||||
}
|
||||
const accepted = new Map(), rejected = { outsideRun: 0, earlier: 0, outOfOrder: 0, noRun: 0 };
|
||||
// 3. Walking down from the newest chapter, release dates never rise again: a lower chapter
|
||||
// dated after a higher one is a back-catalog re-release (One Piece ch. 8 on 2019-02-06).
|
||||
let ceiling = Infinity;
|
||||
for (const [key, item] of [...official].sort((a, b) => Number(b[0]) - Number(a[0]))) {
|
||||
if (!run || !run.start) { rejected.noRun++; continue; }
|
||||
if (item.time < run.start - 7 * DAY || (run.end && item.time > run.end + 30 * DAY)) { rejected.outsideRun++; continue; }
|
||||
const earliest = Math.min(scans.get(key)?.time ?? Infinity, evidence.get(key) ?? Infinity);
|
||||
if (earliest < item.time - EARLY_TOLERANCE_DAYS * DAY) { rejected.earlier++; continue; }
|
||||
if (item.time > ceiling + ORDER_TOLERANCE_DAYS * DAY) { rejected.outOfOrder++; continue; }
|
||||
ceiling = Math.min(ceiling, item.time);
|
||||
accepted.set(key, { at: item.date, precision: 'day', source: `MangaUpdates · ${item.groups.join(', ')}` });
|
||||
}
|
||||
return { accepted, rejected, officialCount: official.size };
|
||||
}
|
||||
|
||||
// Chapter version folders of one manga with their numeric chapter key.
|
||||
async function chapterVersions(mangaDir) {
|
||||
const out = [];
|
||||
const dirs = async dir => (await fs.readdir(dir, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name);
|
||||
for (const book of await dirs(mangaDir)) {
|
||||
if (!parseKey(book)) continue;
|
||||
for (const chapter of await dirs(path.join(mangaDir, book))) {
|
||||
const parsed = parseKey(chapter);
|
||||
if (!parsed || parsed.x) continue; // extras and specials have no chapter number to match
|
||||
const file = path.join(mangaDir, book, chapter, 'chapter.json');
|
||||
let meta;
|
||||
try { meta = JSON.parse(await fs.readFile(file, 'utf8')); } catch { continue; }
|
||||
out.push({ key: chapterKey(parsed.num), book, file, meta });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Look up, decide and (with apply) write x-dates.release for one manga folder.
|
||||
async function refreshMangaReleaseDates(mangaDir, sources, { apply = false, pages = Infinity, maxAgeMs } = {}) {
|
||||
const meta = JSON.parse(await fs.readFile(path.join(mangaDir, 'manga.json'), 'utf8').catch(() => '{}'));
|
||||
const versions = await chapterVersions(mangaDir);
|
||||
const summary = { manga: path.basename(mangaDir), series: null, chapters: new Set(versions.map(v => v.key)).size, official: 0, accepted: 0, written: 0, cleared: 0, rejected: null, ambiguous: false };
|
||||
// Seasons that restart at chapter 1 cannot be matched to MangaUpdates' running numbers.
|
||||
const books = new Map();
|
||||
for (const version of versions) books.set(version.key, (books.get(version.key) || new Set()).add(version.book));
|
||||
if ([...books.values()].some(set => set.size > 1)) { summary.ambiguous = true; return summary; }
|
||||
const series = await sources.findSeries(meta);
|
||||
if (!series) return summary;
|
||||
summary.series = series;
|
||||
const run = await sources.originalRun(meta['x-anilist']?.id || meta['x-scraper']?.ids?.anilist);
|
||||
const evidence = new Map();
|
||||
for (const version of versions) {
|
||||
const time = Date.parse(version.meta['x-dates']?.source?.at);
|
||||
if (version.key && Number.isFinite(time) && time < (evidence.get(version.key) ?? Infinity)) evidence.set(version.key, time);
|
||||
}
|
||||
const { accepted, rejected, officialCount } = officialDates(await sources.releases(series.id, { pages, ...(maxAgeMs ? { maxAgeMs } : {}) }), run, evidence);
|
||||
summary.official = officialCount; summary.rejected = rejected;
|
||||
const have = new Set(versions.map(v => v.key));
|
||||
summary.accepted = [...accepted.keys()].filter(key => have.has(key)).length;
|
||||
for (const version of versions) {
|
||||
const dates = version.meta['x-dates'] || { version: 1 }, next = accepted.get(version.key);
|
||||
const prior = dates.release;
|
||||
let changed = false;
|
||||
if (next && JSON.stringify(prior) !== JSON.stringify(next)) { dates.release = next; changed = true; summary.written++; }
|
||||
// A date this job wrote earlier and no longer accepts is withdrawn; other release dates stay.
|
||||
else if (!next && prior && /^MangaUpdates\b/.test(prior.source || '') && pages === Infinity) { delete dates.release; changed = true; summary.cleared++; }
|
||||
if (changed && apply) await writeJsonAtomic(version.file, { ...version.meta, 'x-dates': dates });
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
module.exports = { ReleaseDateSources, officialDates, refreshMangaReleaseDates, chapterKey, isOfficial, EARLY_TOLERANCE_DAYS };
|
||||
+26
-1
@@ -13,6 +13,7 @@ const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, summarizeInv
|
||||
const { inspectLegacyWork, promoteLegacyWork } = require('./lib/legacy-publication');
|
||||
const { classifyFailure, safeMessage, progressKey } = require('./lib/failures');
|
||||
const { writeJsonAtomic } = require('./lib/json-file');
|
||||
const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates');
|
||||
|
||||
const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs'];
|
||||
const FAST_RATE_SETTINGS = {
|
||||
@@ -353,12 +354,36 @@ class ScraperService {
|
||||
const delay = Math.max(1000, next.getTime() - Date.now());
|
||||
this.discoveryTimer = setTimeout(() => {
|
||||
this.discoveryTimer = null;
|
||||
this.enqueueDiscoveryChecks('schedule').catch(error => this.event('error', `Nightly discovery failed: ${error.message}`))
|
||||
this.refreshReleaseDates('schedule').catch(error => this.event('error', `Nightly release dates failed: ${error.message}`))
|
||||
.then(() => this.enqueueDiscoveryChecks('schedule'))
|
||||
.catch(error => this.event('error', `Nightly discovery failed: ${error.message}`))
|
||||
.finally(() => { if (!this.stopping) this.scheduleDiscovery(); });
|
||||
}, delay);
|
||||
this.discoveryTimer.unref();
|
||||
}
|
||||
|
||||
// Nightly, before the update checks: official release dates of ongoing series from their
|
||||
// newest MangaUpdates releases (BL-0048). Runs only while no download is active, so the
|
||||
// scraper stays the single writer of chapter.json.
|
||||
async refreshReleaseDates(reason = 'manual') {
|
||||
if (this.active.size) { await this.event('info', 'Release dates skipped: a download is running', { reason }); return null; }
|
||||
const inventory = await this.libraryInventory(true);
|
||||
const sources = new ReleaseDateSources({ cacheDir: path.join(this.dataRoot, 'release-dates-cache') });
|
||||
let checked = 0, written = 0;
|
||||
for (const manga of inventory.mangas) {
|
||||
if (this.stopping || this.active.size) break;
|
||||
if (manga.meta?.status !== 'ongoing') continue;
|
||||
try {
|
||||
const result = await refreshMangaReleaseDates(manga.path, sources, { apply: true, pages: 1, maxAgeMs: 20 * 3600e3 });
|
||||
checked++; written += result.written;
|
||||
} catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); }
|
||||
}
|
||||
// The reader reconciles fully when the revision rises without a visible folder change.
|
||||
if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
|
||||
await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written });
|
||||
return { checked, written };
|
||||
}
|
||||
|
||||
pendingSeries(source, seriesId) {
|
||||
return this.state.queue.some(item => item.source === source && String(item.seriesId) === String(seriesId)
|
||||
&& ['queued', 'downloading', 'discovering', 'ready', 'error', 'retry_wait', 'quarantined', 'unavailable'].includes(item.status));
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs').promises;
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { officialDates, refreshMangaReleaseDates, chapterKey } = require('../metadata/release-dates');
|
||||
|
||||
const DAY = 864e5;
|
||||
const run = (start, end = null) => ({ start: Date.parse(start), end: end ? Date.parse(end) : null });
|
||||
const viz = (chapter, date) => ({ chapter, date, groups: ['Viz', 'MANGA Plus'] });
|
||||
const scan = (chapter, date) => ({ chapter, date, groups: ['Some Scans'] });
|
||||
|
||||
test('chapter keys only match single numbered chapters', () => {
|
||||
assert.equal(chapterKey('001'), '1');
|
||||
assert.equal(chapterKey('10.50'), '10.5');
|
||||
assert.equal(chapterKey('1-3'), null);
|
||||
assert.equal(chapterKey('7a'), null);
|
||||
});
|
||||
|
||||
test('simulpub dates are accepted, re-releases after the run are not', () => {
|
||||
const ongoing = officialDates([viz('1', '2023-09-17'), viz('2', '2023-09-24'), scan('2', '2023-09-23')], run('2023-09-18'));
|
||||
assert.deepEqual([...ongoing.accepted.keys()].sort(), ['1', '2'], 'scans a day early do not disqualify');
|
||||
assert.equal(ongoing.accepted.get('1').at, '2023-09-17');
|
||||
assert.equal(ongoing.accepted.get('1').source, 'MangaUpdates · Viz, MANGA Plus');
|
||||
const ended = officialDates([viz('700', '2020-12-20')], run('1999-09-21', '2014-11-10'));
|
||||
assert.equal(ended.accepted.size, 0);
|
||||
assert.equal(ended.rejected.outsideRun, 1);
|
||||
assert.equal(officialDates([viz('1', '2020-01-01')], null).rejected.noRun, 1);
|
||||
});
|
||||
|
||||
test('an earlier scanlation or source upload means the official date is a re-release', () => {
|
||||
const scanned = officialDates([viz('5', '2021-06-01'), scan('5', '2019-01-01')], run('2018-01-01'));
|
||||
assert.equal(scanned.rejected.earlier, 1);
|
||||
const uploaded = officialDates([viz('5', '2021-06-01')], run('2018-01-01'), new Map([['5', Date.parse('2021-05-01')]]));
|
||||
assert.equal(uploaded.rejected.earlier, 1);
|
||||
const leak = officialDates([viz('5', '2021-06-04')], run('2018-01-01'), new Map([['5', Date.parse('2021-06-04') - 2 * DAY]]));
|
||||
assert.equal(leak.accepted.size, 1, 'an upload two days early is a leak, not an older release');
|
||||
});
|
||||
|
||||
test('back-catalog chapters dated after newer chapters are out of order', () => {
|
||||
const result = officialDates([
|
||||
viz('8', '2019-02-06'), viz('33', '2019-07-31'), // weekly back-catalog re-release
|
||||
viz('928', '2018-12-21'), viz('930', '2019-01-20'), viz('931', '2019-02-03'), viz('1196', '2026-10-18'),
|
||||
viz('929', '2019-01-21'), // a day after 930: a double release is still in order
|
||||
], run('1997-07-22'));
|
||||
assert.deepEqual([...result.accepted.keys()].sort((a, b) => a - b), ['928', '929', '930', '931', '1196']);
|
||||
assert.equal(result.rejected.outOfOrder, 2);
|
||||
});
|
||||
|
||||
async function library(chapters) {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'release-dates-'));
|
||||
const manga = path.join(root, 'Kagurabachi');
|
||||
await fs.mkdir(manga, { recursive: true });
|
||||
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Kagurabachi', year: 2023, 'x-anilist': { id: 169355 } }));
|
||||
for (const [folder, meta] of Object.entries(chapters)) {
|
||||
await fs.mkdir(path.join(manga, folder), { recursive: true });
|
||||
await fs.writeFile(path.join(manga, folder, 'chapter.json'), JSON.stringify(meta));
|
||||
}
|
||||
return manga;
|
||||
}
|
||||
const fakeSources = releases => ({
|
||||
async findSeries(meta) { return meta.title === 'Kagurabachi' ? { id: '60441951323', title: 'Kagurabachi' } : null; },
|
||||
async originalRun() { return run('2023-09-18'); },
|
||||
async releases() { return releases; },
|
||||
});
|
||||
|
||||
test('refresh writes x-dates.release, keeps other dates and withdraws its own stale dates', async () => {
|
||||
const manga = await library({
|
||||
'v01/c0001 [WeebCentral]': { title: '', added: '2026-10-01T00:00:00Z', 'x-dates': { version: 1, source: { at: '2024-09-07T10:00:00Z', name: 'WeebCentral', batch: true } } },
|
||||
'v01/c0002 [WeebCentral]': { title: '', 'x-dates': { version: 1, release: { at: '2020-01-01', precision: 'day', source: 'MangaUpdates · Viz' } } },
|
||||
'v01/c0003 [WeebCentral]': { title: '', 'x-dates': { version: 1, release: { at: '2023-10-01', precision: 'day', source: 'Hand-entered' } } },
|
||||
});
|
||||
const dry = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')]));
|
||||
assert.equal(dry.written, 1);
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0001 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release, undefined, 'dry run writes nothing');
|
||||
const applied = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')]), { apply: true });
|
||||
assert.deepEqual([applied.written, applied.cleared], [1, 1]);
|
||||
const one = JSON.parse(await fs.readFile(path.join(manga, 'v01/c0001 [WeebCentral]/chapter.json'), 'utf8'));
|
||||
assert.deepEqual(one['x-dates'].release, { at: '2023-09-17', precision: 'day', source: 'MangaUpdates · Viz, MANGA Plus' });
|
||||
assert.equal(one['x-dates'].source.batch, true);
|
||||
assert.equal(one.added, '2026-10-01T00:00:00Z');
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0002 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release, undefined);
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0003 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release.source, 'Hand-entered');
|
||||
});
|
||||
|
||||
test('seasons that restart their numbering are skipped', async () => {
|
||||
const manga = await library({ 'v01 Season 1/c0001': { title: '' }, 'v02 Season 2/c0001': { title: '' } });
|
||||
const result = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')]), { apply: true });
|
||||
assert.equal(result.ambiguous, true);
|
||||
assert.equal(result.written, 0);
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// Official chapter release dates from MangaUpdates (BL-0048) for a whole library or one manga.
|
||||
// Dry run by default. Apply only with the scraper stopped (--apply --writers-stopped), then let a
|
||||
// full reconciliation (or admin Rescan) pick up the edited chapter.json files.
|
||||
const fs = require('node:fs').promises;
|
||||
const path = require('node:path');
|
||||
const { ReleaseDateSources, refreshMangaReleaseDates } = require('../scraper/metadata/release-dates');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const apply = args.includes('--apply');
|
||||
const only = args.find(arg => arg.startsWith('--manga='))?.slice('--manga='.length);
|
||||
const cacheDir = args.find(arg => arg.startsWith('--cache='))?.slice('--cache='.length) || path.join(require('node:os').tmpdir(), 'mangacircus-release-dates');
|
||||
const root = args.find(arg => !arg.startsWith('--'));
|
||||
if (!root || args.some(arg => arg.startsWith('--') && !['--apply', '--writers-stopped'].includes(arg) && !/^--(manga|cache)=/.test(arg))
|
||||
|| (apply && !args.includes('--writers-stopped'))) {
|
||||
console.error('Usage: node tools/release-dates.js <library> [--manga=<folder>] [--cache=<dir>] [--apply --writers-stopped]');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const sources = new ReleaseDateSources({ cacheDir });
|
||||
const totals = { manga: 0, matched: 0, chapters: 0, accepted: 0, written: 0, cleared: 0, outsideRun: 0, earlier: 0, outOfOrder: 0, noRun: 0 };
|
||||
const names = (await fs.readdir(root, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name).sort();
|
||||
for (const name of names) {
|
||||
if (only && name !== only) continue;
|
||||
try {
|
||||
const s = await refreshMangaReleaseDates(path.join(root, name), sources, { apply });
|
||||
totals.manga++; totals.chapters += s.chapters;
|
||||
if (s.ambiguous) { console.log(`${name.slice(0, 44).padEnd(44)} skipped: chapter numbers repeat across books`); continue; }
|
||||
if (!s.series) { console.log(`${name.slice(0, 44).padEnd(44)} no MangaUpdates match`); continue; }
|
||||
totals.matched++; totals.accepted += s.accepted; totals.written += s.written; totals.cleared += s.cleared;
|
||||
for (const k of ['outsideRun', 'earlier', 'outOfOrder', 'noRun']) totals[k] += s.rejected[k];
|
||||
console.log(`${name.slice(0, 44).padEnd(44)} ${String(s.chapters).padStart(5)} ch official ${String(s.official).padStart(4)} accepted ${String(s.accepted).padStart(4)}`
|
||||
+ ` rejected: outside run ${s.rejected.outsideRun}, earlier evidence ${s.rejected.earlier}, out of order ${s.rejected.outOfOrder}`
|
||||
+ `${s.rejected.noRun ? `, no AniList run ${s.rejected.noRun}` : ''} (${s.series.title})`);
|
||||
} catch (error) { console.warn(`${name}: ${error.message}`); }
|
||||
}
|
||||
console.log(JSON.stringify({ apply, ...totals }));
|
||||
}
|
||||
|
||||
main().catch(error => { console.error(error); process.exit(1); });
|
||||
Reference in new issue
Block a user