diff --git a/docs/library-format.md b/docs/library-format.md index d40d648..1f7a261 100644 --- a/docs/library-format.md +++ b/docs/library-format.md @@ -229,8 +229,22 @@ An AniList refresh with a different ID is stopped for review before chapter publ - `source`: when the source site uploaded this version (WeebCentral's full chapter list, MangaDex `publishAt`). `batch: true` means it arrived with 3 or more other uploads less than 10 minutes apart, which is a site import, not a release. -- `release`: the official public release (`precision` `day`, `month` or `year`). Not - filled yet; a metadata job will add it. +- `release`: the official public release (`precision` `day`, `month` or `year`). It is + filled from MangaUpdates releases by publishers such as Viz, MANGA Plus or Webtoon + (`scraper/metadata/release-dates.js`). Those lists include English re-releases of old + chapters, so a date is kept only when all of these hold: + - it lies within the work's original run on AniList; + - no scanlation or source upload shows the chapter more than 4 days earlier; + - it fits the release order: walking down from the newest chapter, a lower chapter + can't be dated more than 2 days after a higher one. + + Series whose chapter numbers restart in a later book are skipped. + - The scraper refreshes ongoing series nightly before its update checks, but only + while no download is running. + - `node tools/release-dates.js ` covers a whole library: a dry run by + default, then `--apply --writers-stopped`. + - A date that this job wrote and no longer accepts is removed. Release dates from any + other source stay. - `ecosystem`: when the chapter first entered any Manga Circus library. Peer transfers carry it, and the earliest known date always wins. A peer's claim can't be earlier than the source upload or later than now. diff --git a/docs/peering.md b/docs/peering.md index 1cae165..3486fcb 100644 --- a/docs/peering.md +++ b/docs/peering.md @@ -118,7 +118,10 @@ date at which any version of that chapter number entered a Manga Circus library to the sender. Older chapters fall back to their `added` date. The receiver stores the date as `x-dates.ecosystem` (`via: "peer "`) and only lets it lower an existing date, never raise it. See `docs/library-format.md`. Older peers omit `dates`, and the -receiver then uses its own arrival time. +receiver then uses its own arrival time. A sync also lowers the ecosystem date of +chapters this library already has, including ones it scraped itself, when the peer +knows an earlier date for the same book and chapter number. The job reports these as +`ecosystemLowered`. The reader shows the new date after its next full reconciliation. | `GET /peer/v1/blobs/{id}` | One image, optional `Range: bytes=N-` and `If-Match`; 206 on resume | Unknown/unshared IDs return 404. Revoked serving returns 403; bad/missing grants diff --git a/peering/adapter.js b/peering/adapter.js index 2d3644d..5dd67b6 100644 --- a/peering/adapter.js +++ b/peering/adapter.js @@ -12,7 +12,8 @@ const { publishManifest } = require('../scraper/lib/publisher'); const { writeJsonAtomic } = require('../scraper/lib/json-file'); const { sleep } = require('../scraper/lib/delay'); const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity'); -const { fromPeer } = require('../chapter-dates'); +const { fromPeer, publishedDates } = require('../chapter-dates'); +const { parseKey, parseSpecialFolder } = require('../chapter-identity'); const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering')); const store = () => new PeeringState(dataRoot()); @@ -28,7 +29,37 @@ const infoFor = value => { }; function refsOf(result) { return result.chapters.map(ref => ({ id: ref.id, number: ref.number, volume: ref.volume, - book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount })); + book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount, + ...(ref.dates && typeof ref.dates === 'object' ? { dates: ref.dates } : {}) })); +} +// Chapters this library already has adopt a peer's earlier ecosystem date (BL-0049): the +// chapter entered Manga Circus there first. Later claims never raise a date. +async function lowerEcosystemDates(mangaPath, refs, alias) { + const claims = new Map(); + for (const ref of refs) { + const claim = fromPeer(ref.dates, alias).ecosystem; + const key = `${ref.volume}/${ref.number}`; + if (claim && (!claims.has(key) || Date.parse(claim.at) < Date.parse(claims.get(key).at))) claims.set(key, claim); + } + let lowered = 0; + if (!claims.size) return lowered; + for (const book of await fs.readdir(mangaPath, { withFileTypes: true })) { + const bookKey = book.isDirectory() && parseKey(book.name)?.key; + if (!bookKey) continue; + for (const chapter of await fs.readdir(path.join(mangaPath, book.name), { withFileTypes: true })) { + const parsed = chapter.isDirectory() && (parseSpecialFolder(chapter.name) || parseKey(chapter.name)); + const claim = parsed && claims.get(`${bookKey}/${parsed.key}`); + if (!claim) continue; + const file = path.join(mangaPath, book.name, chapter.name, 'chapter.json'); + let meta; + try { meta = JSON.parse(await fs.readFile(file, 'utf8')); } catch { continue; } + const next = publishedDates(meta, { ecosystem: claim }); + if (next['x-dates'].ecosystem.at === meta['x-dates']?.ecosystem?.at) continue; + await writeJsonAtomic(file, { ...meta, ...next }); + lowered++; + } + } + return lowered; } function sameTitle(left, right) { return String(left || '').normalize('NFKC').trim().replace(/\s+/g, ' ').toLowerCase() @@ -102,6 +133,7 @@ async function run(info, item, options) { await writeJsonAtomic(path.join(manga.path, 'manga.json'), manga.meta); } const selected = selectByScope(catalog.chapters.filter(ref => shouldDownload(manga, ref, source, item.existingPolicy)), item.scope); + const ecosystemLowered = manga ? await lowerEcosystemDates(manga.path, catalog.chapters, peer.alias) : 0; job.total = selected.length; job.jobCompleted = 0; job.current = null; await record(); let publishedCount = 0; const identityToPersist = localIdentity || workIdentity; @@ -150,7 +182,7 @@ async function run(info, item, options) { if (index < selected.length - 1) { stop(); await sleep(Math.max(500, Number(options.settings.chapterMinMs || 500))); } } job.status = 'complete'; job.current = null; await record(); - return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length }; + return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length, ecosystemLowered }; } function adapterForPeer(value) { const info = infoFor(value); diff --git a/peering/test/peering.test.js b/peering/test/peering.test.js index b2255b3..3416ad2 100644 --- a/peering/test/peering.test.js +++ b/peering/test/peering.test.js @@ -142,9 +142,15 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t 'The chapter keeps the date it first entered Manga Circus on A, not its arrival on B'); assert.equal(received['x-dates'].source.at, '2026-09-30T15:00:00.000Z'); assert.notEqual(received.added, '2026-10-01T08:00:00.000Z', '"added" is the arrival in B\'s own library'); + // B claims it got the chapter later on its own: the next sync lowers it to A's date again. + await fsp.writeFile(path.join(receivedDir, 'chapter.json'), JSON.stringify({ ...received, + 'x-dates': { ...received['x-dates'], ecosystem: { at: '2026-10-20T00:00:00.000Z', via: 'this server' } } })); const second = await run(info, { scope: { mode: 'missing' }, existingPolicy: 'skip-any' }, { libraryRoot: libB, settings: { chapterMinMs: 500 }, shouldStop: () => false }); assert.equal(second.publishedCount, 0); + assert.equal(second.ecosystemLowered, 1, 'An existing chapter adopts the peer\'s earlier ecosystem date'); + assert.deepEqual(JSON.parse(await fsp.readFile(path.join(receivedDir, 'chapter.json'), 'utf8'))['x-dates'].ecosystem, + { at: '2026-10-01T08:00:00.000Z', via: 'peer ServerA' }); // Simulate B adding one chapter to its imported edition, then sharing it back. // A must identify its original folder by the carried origin identity and fetch diff --git a/scraper/metadata/release-dates.js b/scraper/metadata/release-dates.js new file mode 100644 index 0000000..9557dfe --- /dev/null +++ b/scraper/metadata/release-dates.js @@ -0,0 +1,180 @@ +'use strict'; +// Official chapter release dates (BL-0048) from MangaUpdates publisher releases. +// MangaUpdates also lists English digital re-releases of old chapters as "official" +// (Naruto ch. 700 shows 2020; it came out in Japan in 2014). A date is accepted only when +// 1. it lies inside the work's original run (AniList start - 7 days ... end + 30 days), and +// 2. nothing shows the chapter earlier: no scanlation release in MangaUpdates and no +// source upload (bulk or not) more than EARLY_TOLERANCE_DAYS before it, and +// 3. it fits the release order (see officialDates). +// Simulpub releases pass all three; back-catalog re-releases fail at least one. +const fs = require('node:fs').promises; +const path = require('node:path'); +const crypto = require('node:crypto'); +const { parseKey } = require('../../chapter-identity'); +const { writeJsonAtomic } = require('../lib/json-file'); + +const DAY = 864e5; +const EARLY_TOLERANCE_DAYS = 4; // early scans and source uploads of weekly series arrive 1-3 days before the official date +const ORDER_TOLERANCE_DAYS = 2; // double releases and day-granular dates may put a lower chapter a day or two later +const OFFICIAL = new Set(['viz', 'viz media', 'manga plus', 'mangaplus', 'shueisha', 'kodansha', 'kodansha usa', 'k manga', + 'yen press', 'seven seas', 'square enix', 'square enix manga', 'manga up!', 'comikey', 'azuki', 'tappytoon', 'webtoon', + 'line webtoon', 'webtoons', 'tapas', 'pocket comics', 'manta', 'lezhin', 'lezhin comics', 'j-novel club', 'dark horse', + 'vertical', 'denpa', 'tokyopop', 'ize press', 'one peace books', 'inky pen', 'kakaopage', 'kakao webtoon', 'naver webtoon']); +const UA = 'MangaCircus/1.0 (self-hosted manga reader; chapter release dates)'; +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const norm = value => String(value || '').toLowerCase().normalize('NFKD').replace(/[^a-z0-9]+/g, ''); +// "001", "1.50" and "1" are the same chapter; ranges, volumes and letters are not single chapters. +const chapterKey = value => (/^\s*\d+(?:\.\d+)?\s*$/.test(String(value ?? '')) ? String(Number(value)) : null); +const isOfficial = groups => groups.some(name => OFFICIAL.has(String(name).toLowerCase().trim())); + +async function cached(cacheDir, key, maxAgeMs, load) { + const file = cacheDir && path.join(cacheDir, crypto.createHash('sha256').update(key).digest('hex').slice(0, 32) + '.json'); + if (file) { + try { + const stat = await fs.stat(file); + if (Date.now() - stat.mtimeMs < maxAgeMs) return JSON.parse(await fs.readFile(file, 'utf8')); + } catch { /* not cached */ } + } + const value = await load(); + if (file) { await fs.mkdir(cacheDir, { recursive: true }); await writeJsonAtomic(file, value); } + return value; +} + +class ReleaseDateSources { + constructor(options = {}) { + this.cacheDir = options.cacheDir || null; + this.minGapMs = options.minGapMs ?? 1100; + this.fetch = options.fetch || globalThis.fetch; + this.last = 0; + } + async request(url, body) { + const wait = this.last + this.minGapMs - Date.now(); + if (wait > 0) await sleep(wait); + this.last = Date.now(); + const response = await this.fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', 'user-agent': UA }, + body: JSON.stringify(body), signal: AbortSignal.timeout(30000) }); + if (!response.ok) throw new Error(`${new URL(url).hostname} HTTP ${response.status}`); + return response.json(); + } + // MangaUpdates series whose matched name equals one of the manga's titles and whose year fits. + async findSeries(meta) { + const titles = [meta.title, ...(Array.isArray(meta.altTitles) ? meta.altTitles : [])].map(norm).filter(Boolean); + const year = Number(meta.year) || null; + const result = await cached(this.cacheDir, `mu-search:${meta.title}`, 30 * DAY, + () => this.request('https://api.mangaupdates.com/v1/series/search', { search: String(meta.title || ''), perpage: 10 })); + for (const hit of result.results || []) { + const record = hit.record || {}, hitYear = Number(record.year) || null; + if (year && hitYear && Math.abs(hitYear - year) > 1) continue; + if ([record.title, hit.hit_title].map(norm).some(name => name && titles.includes(name))) { + return { id: String(record.series_id), title: record.title, url: record.url || null }; + } + } + return null; + } + async releases(seriesId, { pages = Infinity, maxAgeMs = 7 * DAY } = {}) { + return cached(this.cacheDir, `mu-releases:${seriesId}:${pages}`, maxAgeMs, async () => { + const rows = []; + for (let page = 1; page <= pages; page++) { + const result = await this.request('https://api.mangaupdates.com/v1/releases/search', + { search: String(seriesId), search_type: 'series', perpage: 100, page, orderby: 'date' }); + for (const { record } of result.results || []) { + if (record?.release_date) rows.push({ chapter: record.chapter, date: record.release_date, groups: (record.groups || []).map(g => g.name) }); + } + if (!result.results?.length || page * 100 >= (result.total_hits || 0)) break; + } + return rows; + }); + } + async originalRun(anilistId) { + if (!anilistId) return null; + const result = await cached(this.cacheDir, `anilist-run:${anilistId}`, 30 * DAY, () => this.request('https://graphql.anilist.co', { + query: 'query ($id: Int) { Media(id: $id, type: MANGA) { status startDate { year month day } endDate { year month day } } }', + variables: { id: Number(anilistId) }, + })); + const media = result.data?.Media; + if (!media) return null; + const date = (d, end) => (d?.year ? Date.UTC(d.year, (d.month || (end ? 12 : 1)) - 1, d.day || (end ? 28 : 1)) : null); + return { status: media.status, start: date(media.startDate, false), end: media.status === 'RELEASING' ? null : date(media.endDate, true) }; + } +} + +// Pure decision: official dates per chapter key, with the reasons others were rejected. +// evidence: Map chapterKey -> earliest other time (ms) the library knows for that chapter. +function officialDates(releases, run, evidence = new Map()) { + const official = new Map(), scans = new Map(); + for (const row of releases) { + const key = chapterKey(row.chapter), time = Date.parse(row.date); + if (!key || !Number.isFinite(time)) continue; + const target = isOfficial(row.groups) ? official : scans; + const prior = target.get(key); + if (!prior || time < prior.time) target.set(key, { time, date: String(row.date).slice(0, 10), groups: row.groups }); + } + const accepted = new Map(), rejected = { outsideRun: 0, earlier: 0, outOfOrder: 0, noRun: 0 }; + // 3. Walking down from the newest chapter, release dates never rise again: a lower chapter + // dated after a higher one is a back-catalog re-release (One Piece ch. 8 on 2019-02-06). + let ceiling = Infinity; + for (const [key, item] of [...official].sort((a, b) => Number(b[0]) - Number(a[0]))) { + if (!run || !run.start) { rejected.noRun++; continue; } + if (item.time < run.start - 7 * DAY || (run.end && item.time > run.end + 30 * DAY)) { rejected.outsideRun++; continue; } + const earliest = Math.min(scans.get(key)?.time ?? Infinity, evidence.get(key) ?? Infinity); + if (earliest < item.time - EARLY_TOLERANCE_DAYS * DAY) { rejected.earlier++; continue; } + if (item.time > ceiling + ORDER_TOLERANCE_DAYS * DAY) { rejected.outOfOrder++; continue; } + ceiling = Math.min(ceiling, item.time); + accepted.set(key, { at: item.date, precision: 'day', source: `MangaUpdates · ${item.groups.join(', ')}` }); + } + return { accepted, rejected, officialCount: official.size }; +} + +// Chapter version folders of one manga with their numeric chapter key. +async function chapterVersions(mangaDir) { + const out = []; + const dirs = async dir => (await fs.readdir(dir, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name); + for (const book of await dirs(mangaDir)) { + if (!parseKey(book)) continue; + for (const chapter of await dirs(path.join(mangaDir, book))) { + const parsed = parseKey(chapter); + if (!parsed || parsed.x) continue; // extras and specials have no chapter number to match + const file = path.join(mangaDir, book, chapter, 'chapter.json'); + let meta; + try { meta = JSON.parse(await fs.readFile(file, 'utf8')); } catch { continue; } + out.push({ key: chapterKey(parsed.num), book, file, meta }); + } + } + return out; +} + +// Look up, decide and (with apply) write x-dates.release for one manga folder. +async function refreshMangaReleaseDates(mangaDir, sources, { apply = false, pages = Infinity, maxAgeMs } = {}) { + const meta = JSON.parse(await fs.readFile(path.join(mangaDir, 'manga.json'), 'utf8').catch(() => '{}')); + const versions = await chapterVersions(mangaDir); + const summary = { manga: path.basename(mangaDir), series: null, chapters: new Set(versions.map(v => v.key)).size, official: 0, accepted: 0, written: 0, cleared: 0, rejected: null, ambiguous: false }; + // Seasons that restart at chapter 1 cannot be matched to MangaUpdates' running numbers. + const books = new Map(); + for (const version of versions) books.set(version.key, (books.get(version.key) || new Set()).add(version.book)); + if ([...books.values()].some(set => set.size > 1)) { summary.ambiguous = true; return summary; } + const series = await sources.findSeries(meta); + if (!series) return summary; + summary.series = series; + const run = await sources.originalRun(meta['x-anilist']?.id || meta['x-scraper']?.ids?.anilist); + const evidence = new Map(); + for (const version of versions) { + const time = Date.parse(version.meta['x-dates']?.source?.at); + if (version.key && Number.isFinite(time) && time < (evidence.get(version.key) ?? Infinity)) evidence.set(version.key, time); + } + const { accepted, rejected, officialCount } = officialDates(await sources.releases(series.id, { pages, ...(maxAgeMs ? { maxAgeMs } : {}) }), run, evidence); + summary.official = officialCount; summary.rejected = rejected; + const have = new Set(versions.map(v => v.key)); + summary.accepted = [...accepted.keys()].filter(key => have.has(key)).length; + for (const version of versions) { + const dates = version.meta['x-dates'] || { version: 1 }, next = accepted.get(version.key); + const prior = dates.release; + let changed = false; + if (next && JSON.stringify(prior) !== JSON.stringify(next)) { dates.release = next; changed = true; summary.written++; } + // A date this job wrote earlier and no longer accepts is withdrawn; other release dates stay. + else if (!next && prior && /^MangaUpdates\b/.test(prior.source || '') && pages === Infinity) { delete dates.release; changed = true; summary.cleared++; } + if (changed && apply) await writeJsonAtomic(version.file, { ...version.meta, 'x-dates': dates }); + } + return summary; +} + +module.exports = { ReleaseDateSources, officialDates, refreshMangaReleaseDates, chapterKey, isOfficial, EARLY_TOLERANCE_DAYS }; diff --git a/scraper/service.js b/scraper/service.js index 1a91fd8..f592f1b 100644 --- a/scraper/service.js +++ b/scraper/service.js @@ -13,6 +13,7 @@ const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, summarizeInv const { inspectLegacyWork, promoteLegacyWork } = require('./lib/legacy-publication'); const { classifyFailure, safeMessage, progressKey } = require('./lib/failures'); const { writeJsonAtomic } = require('./lib/json-file'); +const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates'); const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs']; const FAST_RATE_SETTINGS = { @@ -353,12 +354,36 @@ class ScraperService { const delay = Math.max(1000, next.getTime() - Date.now()); this.discoveryTimer = setTimeout(() => { this.discoveryTimer = null; - this.enqueueDiscoveryChecks('schedule').catch(error => this.event('error', `Nightly discovery failed: ${error.message}`)) + this.refreshReleaseDates('schedule').catch(error => this.event('error', `Nightly release dates failed: ${error.message}`)) + .then(() => this.enqueueDiscoveryChecks('schedule')) + .catch(error => this.event('error', `Nightly discovery failed: ${error.message}`)) .finally(() => { if (!this.stopping) this.scheduleDiscovery(); }); }, delay); this.discoveryTimer.unref(); } + // Nightly, before the update checks: official release dates of ongoing series from their + // newest MangaUpdates releases (BL-0048). Runs only while no download is active, so the + // scraper stays the single writer of chapter.json. + async refreshReleaseDates(reason = 'manual') { + if (this.active.size) { await this.event('info', 'Release dates skipped: a download is running', { reason }); return null; } + const inventory = await this.libraryInventory(true); + const sources = new ReleaseDateSources({ cacheDir: path.join(this.dataRoot, 'release-dates-cache') }); + let checked = 0, written = 0; + for (const manga of inventory.mangas) { + if (this.stopping || this.active.size) break; + if (manga.meta?.status !== 'ongoing') continue; + try { + const result = await refreshMangaReleaseDates(manga.path, sources, { apply: true, pages: 1, maxAgeMs: 20 * 3600e3 }); + checked++; written += result.written; + } catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); } + } + // The reader reconciles fully when the revision rises without a visible folder change. + if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); } + await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written }); + return { checked, written }; + } + pendingSeries(source, seriesId) { return this.state.queue.some(item => item.source === source && String(item.seriesId) === String(seriesId) && ['queued', 'downloading', 'discovering', 'ready', 'error', 'retry_wait', 'quarantined', 'unavailable'].includes(item.status)); diff --git a/scraper/test/release-dates.test.js b/scraper/test/release-dates.test.js new file mode 100644 index 0000000..e677508 --- /dev/null +++ b/scraper/test/release-dates.test.js @@ -0,0 +1,93 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs').promises; +const os = require('node:os'); +const path = require('node:path'); +const { officialDates, refreshMangaReleaseDates, chapterKey } = require('../metadata/release-dates'); + +const DAY = 864e5; +const run = (start, end = null) => ({ start: Date.parse(start), end: end ? Date.parse(end) : null }); +const viz = (chapter, date) => ({ chapter, date, groups: ['Viz', 'MANGA Plus'] }); +const scan = (chapter, date) => ({ chapter, date, groups: ['Some Scans'] }); + +test('chapter keys only match single numbered chapters', () => { + assert.equal(chapterKey('001'), '1'); + assert.equal(chapterKey('10.50'), '10.5'); + assert.equal(chapterKey('1-3'), null); + assert.equal(chapterKey('7a'), null); +}); + +test('simulpub dates are accepted, re-releases after the run are not', () => { + const ongoing = officialDates([viz('1', '2023-09-17'), viz('2', '2023-09-24'), scan('2', '2023-09-23')], run('2023-09-18')); + assert.deepEqual([...ongoing.accepted.keys()].sort(), ['1', '2'], 'scans a day early do not disqualify'); + assert.equal(ongoing.accepted.get('1').at, '2023-09-17'); + assert.equal(ongoing.accepted.get('1').source, 'MangaUpdates · Viz, MANGA Plus'); + const ended = officialDates([viz('700', '2020-12-20')], run('1999-09-21', '2014-11-10')); + assert.equal(ended.accepted.size, 0); + assert.equal(ended.rejected.outsideRun, 1); + assert.equal(officialDates([viz('1', '2020-01-01')], null).rejected.noRun, 1); +}); + +test('an earlier scanlation or source upload means the official date is a re-release', () => { + const scanned = officialDates([viz('5', '2021-06-01'), scan('5', '2019-01-01')], run('2018-01-01')); + assert.equal(scanned.rejected.earlier, 1); + const uploaded = officialDates([viz('5', '2021-06-01')], run('2018-01-01'), new Map([['5', Date.parse('2021-05-01')]])); + assert.equal(uploaded.rejected.earlier, 1); + const leak = officialDates([viz('5', '2021-06-04')], run('2018-01-01'), new Map([['5', Date.parse('2021-06-04') - 2 * DAY]])); + assert.equal(leak.accepted.size, 1, 'an upload two days early is a leak, not an older release'); +}); + +test('back-catalog chapters dated after newer chapters are out of order', () => { + const result = officialDates([ + viz('8', '2019-02-06'), viz('33', '2019-07-31'), // weekly back-catalog re-release + viz('928', '2018-12-21'), viz('930', '2019-01-20'), viz('931', '2019-02-03'), viz('1196', '2026-10-18'), + viz('929', '2019-01-21'), // a day after 930: a double release is still in order + ], run('1997-07-22')); + assert.deepEqual([...result.accepted.keys()].sort((a, b) => a - b), ['928', '929', '930', '931', '1196']); + assert.equal(result.rejected.outOfOrder, 2); +}); + +async function library(chapters) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'release-dates-')); + const manga = path.join(root, 'Kagurabachi'); + await fs.mkdir(manga, { recursive: true }); + await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Kagurabachi', year: 2023, 'x-anilist': { id: 169355 } })); + for (const [folder, meta] of Object.entries(chapters)) { + await fs.mkdir(path.join(manga, folder), { recursive: true }); + await fs.writeFile(path.join(manga, folder, 'chapter.json'), JSON.stringify(meta)); + } + return manga; +} +const fakeSources = releases => ({ + async findSeries(meta) { return meta.title === 'Kagurabachi' ? { id: '60441951323', title: 'Kagurabachi' } : null; }, + async originalRun() { return run('2023-09-18'); }, + async releases() { return releases; }, +}); + +test('refresh writes x-dates.release, keeps other dates and withdraws its own stale dates', async () => { + const manga = await library({ + 'v01/c0001 [WeebCentral]': { title: '', added: '2026-10-01T00:00:00Z', 'x-dates': { version: 1, source: { at: '2024-09-07T10:00:00Z', name: 'WeebCentral', batch: true } } }, + 'v01/c0002 [WeebCentral]': { title: '', 'x-dates': { version: 1, release: { at: '2020-01-01', precision: 'day', source: 'MangaUpdates · Viz' } } }, + 'v01/c0003 [WeebCentral]': { title: '', 'x-dates': { version: 1, release: { at: '2023-10-01', precision: 'day', source: 'Hand-entered' } } }, + }); + const dry = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')])); + assert.equal(dry.written, 1); + assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0001 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release, undefined, 'dry run writes nothing'); + const applied = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')]), { apply: true }); + assert.deepEqual([applied.written, applied.cleared], [1, 1]); + const one = JSON.parse(await fs.readFile(path.join(manga, 'v01/c0001 [WeebCentral]/chapter.json'), 'utf8')); + assert.deepEqual(one['x-dates'].release, { at: '2023-09-17', precision: 'day', source: 'MangaUpdates · Viz, MANGA Plus' }); + assert.equal(one['x-dates'].source.batch, true); + assert.equal(one.added, '2026-10-01T00:00:00Z'); + assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0002 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release, undefined); + assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0003 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release.source, 'Hand-entered'); +}); + +test('seasons that restart their numbering are skipped', async () => { + const manga = await library({ 'v01 Season 1/c0001': { title: '' }, 'v02 Season 2/c0001': { title: '' } }); + const result = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')]), { apply: true }); + assert.equal(result.ambiguous, true); + assert.equal(result.written, 0); +}); diff --git a/tools/release-dates.js b/tools/release-dates.js new file mode 100644 index 0000000..b825b81 --- /dev/null +++ b/tools/release-dates.js @@ -0,0 +1,42 @@ +#!/usr/bin/env node +'use strict'; +// Official chapter release dates from MangaUpdates (BL-0048) for a whole library or one manga. +// Dry run by default. Apply only with the scraper stopped (--apply --writers-stopped), then let a +// full reconciliation (or admin Rescan) pick up the edited chapter.json files. +const fs = require('node:fs').promises; +const path = require('node:path'); +const { ReleaseDateSources, refreshMangaReleaseDates } = require('../scraper/metadata/release-dates'); + +const args = process.argv.slice(2); +const apply = args.includes('--apply'); +const only = args.find(arg => arg.startsWith('--manga='))?.slice('--manga='.length); +const cacheDir = args.find(arg => arg.startsWith('--cache='))?.slice('--cache='.length) || path.join(require('node:os').tmpdir(), 'mangacircus-release-dates'); +const root = args.find(arg => !arg.startsWith('--')); +if (!root || args.some(arg => arg.startsWith('--') && !['--apply', '--writers-stopped'].includes(arg) && !/^--(manga|cache)=/.test(arg)) + || (apply && !args.includes('--writers-stopped'))) { + console.error('Usage: node tools/release-dates.js [--manga=] [--cache=] [--apply --writers-stopped]'); + process.exit(2); +} + +async function main() { + const sources = new ReleaseDateSources({ cacheDir }); + const totals = { manga: 0, matched: 0, chapters: 0, accepted: 0, written: 0, cleared: 0, outsideRun: 0, earlier: 0, outOfOrder: 0, noRun: 0 }; + const names = (await fs.readdir(root, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name).sort(); + for (const name of names) { + if (only && name !== only) continue; + try { + const s = await refreshMangaReleaseDates(path.join(root, name), sources, { apply }); + totals.manga++; totals.chapters += s.chapters; + if (s.ambiguous) { console.log(`${name.slice(0, 44).padEnd(44)} skipped: chapter numbers repeat across books`); continue; } + if (!s.series) { console.log(`${name.slice(0, 44).padEnd(44)} no MangaUpdates match`); continue; } + totals.matched++; totals.accepted += s.accepted; totals.written += s.written; totals.cleared += s.cleared; + for (const k of ['outsideRun', 'earlier', 'outOfOrder', 'noRun']) totals[k] += s.rejected[k]; + console.log(`${name.slice(0, 44).padEnd(44)} ${String(s.chapters).padStart(5)} ch official ${String(s.official).padStart(4)} accepted ${String(s.accepted).padStart(4)}` + + ` rejected: outside run ${s.rejected.outsideRun}, earlier evidence ${s.rejected.earlier}, out of order ${s.rejected.outOfOrder}` + + `${s.rejected.noRun ? `, no AniList run ${s.rejected.noRun}` : ''} (${s.series.title})`); + } catch (error) { console.warn(`${name}: ${error.message}`); } + } + console.log(JSON.stringify({ apply, ...totals })); +} + +main().catch(error => { console.error(error); process.exit(1); });