Official release dates from MangaUpdates; peers lower ecosystem dates (BL-0048, BL-0049)

- scraper/metadata/release-dates.js fills x-dates.release from MangaUpdates
  publisher releases. MangaUpdates also lists English back-catalog re-releases,
  so a date is kept only inside the AniList original run, when no scanlation or
  source upload is more than 4 days earlier, and when it fits the release order
  (no lower chapter dated after a higher one). Seasons that restart chapter
  numbers are skipped; dates this job wrote and no longer accepts are withdrawn.
- The scraper refreshes ongoing series nightly before its update checks, only
  while no download runs, and bumps the publish revision so the reader reconciles.
- tools/release-dates.js for a whole library (dry run default). Production dry
  run: 728 chapters dated (Kagurabachi, JJK, Chainsaw Man, One Piece from 928).
- Peer sync: chapters this library already has adopt a peer's earlier ecosystem
  date (job result ecosystemLowered); the peer listing now carries dates.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scraper/metadata/release-dates.js

@@ -0,0 +1,180 @@
+'use strict';
+// Official chapter release dates (BL-0048) from MangaUpdates publisher releases.
+// MangaUpdates also lists English digital re-releases of old chapters as "official"
+// (Naruto ch. 700 shows 2020; it came out in Japan in 2014). A date is accepted only when
+//   1. it lies inside the work's original run (AniList start - 7 days ... end + 30 days), and
+//   2. nothing shows the chapter earlier: no scanlation release in MangaUpdates and no
+//      source upload (bulk or not) more than EARLY_TOLERANCE_DAYS before it, and
+//   3. it fits the release order (see officialDates).
+// Simulpub releases pass all three; back-catalog re-releases fail at least one.
+const fs = require('node:fs').promises;
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { parseKey } = require('../../chapter-identity');
+const { writeJsonAtomic } = require('../lib/json-file');
+
+const DAY = 864e5;
+const EARLY_TOLERANCE_DAYS = 4; // early scans and source uploads of weekly series arrive 1-3 days before the official date
+const ORDER_TOLERANCE_DAYS = 2; // double releases and day-granular dates may put a lower chapter a day or two later
+const OFFICIAL = new Set(['viz', 'viz media', 'manga plus', 'mangaplus', 'shueisha', 'kodansha', 'kodansha usa', 'k manga',
+  'yen press', 'seven seas', 'square enix', 'square enix manga', 'manga up!', 'comikey', 'azuki', 'tappytoon', 'webtoon',
+  'line webtoon', 'webtoons', 'tapas', 'pocket comics', 'manta', 'lezhin', 'lezhin comics', 'j-novel club', 'dark horse',
+  'vertical', 'denpa', 'tokyopop', 'ize press', 'one peace books', 'inky pen', 'kakaopage', 'kakao webtoon', 'naver webtoon']);
+const UA = 'MangaCircus/1.0 (self-hosted manga reader; chapter release dates)';
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+const norm = value => String(value || '').toLowerCase().normalize('NFKD').replace(/[^a-z0-9]+/g, '');
+// "001", "1.50" and "1" are the same chapter; ranges, volumes and letters are not single chapters.
+const chapterKey = value => (/^\s*\d+(?:\.\d+)?\s*$/.test(String(value ?? '')) ? String(Number(value)) : null);
+const isOfficial = groups => groups.some(name => OFFICIAL.has(String(name).toLowerCase().trim()));
+
+async function cached(cacheDir, key, maxAgeMs, load) {
+  const file = cacheDir && path.join(cacheDir, crypto.createHash('sha256').update(key).digest('hex').slice(0, 32) + '.json');
+  if (file) {
+    try {
+      const stat = await fs.stat(file);
+      if (Date.now() - stat.mtimeMs < maxAgeMs) return JSON.parse(await fs.readFile(file, 'utf8'));
+    } catch { /* not cached */ }
+  }
+  const value = await load();
+  if (file) { await fs.mkdir(cacheDir, { recursive: true }); await writeJsonAtomic(file, value); }
+  return value;
+}
+
+class ReleaseDateSources {
+  constructor(options = {}) {
+    this.cacheDir = options.cacheDir || null;
+    this.minGapMs = options.minGapMs ?? 1100;
+    this.fetch = options.fetch || globalThis.fetch;
+    this.last = 0;
+  }
+  async request(url, body) {
+    const wait = this.last + this.minGapMs - Date.now();
+    if (wait > 0) await sleep(wait);
+    this.last = Date.now();
+    const response = await this.fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', 'user-agent': UA },
+      body: JSON.stringify(body), signal: AbortSignal.timeout(30000) });
+    if (!response.ok) throw new Error(`${new URL(url).hostname} HTTP ${response.status}`);
+    return response.json();
+  }
+  // MangaUpdates series whose matched name equals one of the manga's titles and whose year fits.
+  async findSeries(meta) {
+    const titles = [meta.title, ...(Array.isArray(meta.altTitles) ? meta.altTitles : [])].map(norm).filter(Boolean);
+    const year = Number(meta.year) || null;
+    const result = await cached(this.cacheDir, `mu-search:${meta.title}`, 30 * DAY,
+      () => this.request('https://api.mangaupdates.com/v1/series/search', { search: String(meta.title || ''), perpage: 10 }));
+    for (const hit of result.results || []) {
+      const record = hit.record || {}, hitYear = Number(record.year) || null;
+      if (year && hitYear && Math.abs(hitYear - year) > 1) continue;
+      if ([record.title, hit.hit_title].map(norm).some(name => name && titles.includes(name))) {
+        return { id: String(record.series_id), title: record.title, url: record.url || null };
+      }
+    }
+    return null;
+  }
+  async releases(seriesId, { pages = Infinity, maxAgeMs = 7 * DAY } = {}) {
+    return cached(this.cacheDir, `mu-releases:${seriesId}:${pages}`, maxAgeMs, async () => {
+      const rows = [];
+      for (let page = 1; page <= pages; page++) {
+        const result = await this.request('https://api.mangaupdates.com/v1/releases/search',
+          { search: String(seriesId), search_type: 'series', perpage: 100, page, orderby: 'date' });
+        for (const { record } of result.results || []) {
+          if (record?.release_date) rows.push({ chapter: record.chapter, date: record.release_date, groups: (record.groups || []).map(g => g.name) });
+        }
+        if (!result.results?.length || page * 100 >= (result.total_hits || 0)) break;
+      }
+      return rows;
+    });
+  }
+  async originalRun(anilistId) {
+    if (!anilistId) return null;
+    const result = await cached(this.cacheDir, `anilist-run:${anilistId}`, 30 * DAY, () => this.request('https://graphql.anilist.co', {
+      query: 'query ($id: Int) { Media(id: $id, type: MANGA) { status startDate { year month day } endDate { year month day } } }',
+      variables: { id: Number(anilistId) },
+    }));
+    const media = result.data?.Media;
+    if (!media) return null;
+    const date = (d, end) => (d?.year ? Date.UTC(d.year, (d.month || (end ? 12 : 1)) - 1, d.day || (end ? 28 : 1)) : null);
+    return { status: media.status, start: date(media.startDate, false), end: media.status === 'RELEASING' ? null : date(media.endDate, true) };
+  }
+}
+
+// Pure decision: official dates per chapter key, with the reasons others were rejected.
+// evidence: Map chapterKey -> earliest other time (ms) the library knows for that chapter.
+function officialDates(releases, run, evidence = new Map()) {
+  const official = new Map(), scans = new Map();
+  for (const row of releases) {
+    const key = chapterKey(row.chapter), time = Date.parse(row.date);
+    if (!key || !Number.isFinite(time)) continue;
+    const target = isOfficial(row.groups) ? official : scans;
+    const prior = target.get(key);
+    if (!prior || time < prior.time) target.set(key, { time, date: String(row.date).slice(0, 10), groups: row.groups });
+  }
+  const accepted = new Map(), rejected = { outsideRun: 0, earlier: 0, outOfOrder: 0, noRun: 0 };
+  // 3. Walking down from the newest chapter, release dates never rise again: a lower chapter
+  //    dated after a higher one is a back-catalog re-release (One Piece ch. 8 on 2019-02-06).
+  let ceiling = Infinity;
+  for (const [key, item] of [...official].sort((a, b) => Number(b[0]) - Number(a[0]))) {
+    if (!run || !run.start) { rejected.noRun++; continue; }
+    if (item.time < run.start - 7 * DAY || (run.end && item.time > run.end + 30 * DAY)) { rejected.outsideRun++; continue; }
+    const earliest = Math.min(scans.get(key)?.time ?? Infinity, evidence.get(key) ?? Infinity);
+    if (earliest < item.time - EARLY_TOLERANCE_DAYS * DAY) { rejected.earlier++; continue; }
+    if (item.time > ceiling + ORDER_TOLERANCE_DAYS * DAY) { rejected.outOfOrder++; continue; }
+    ceiling = Math.min(ceiling, item.time);
+    accepted.set(key, { at: item.date, precision: 'day', source: `MangaUpdates · ${item.groups.join(', ')}` });
+  }
+  return { accepted, rejected, officialCount: official.size };
+}
+
+// Chapter version folders of one manga with their numeric chapter key.
+async function chapterVersions(mangaDir) {
+  const out = [];
+  const dirs = async dir => (await fs.readdir(dir, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name);
+  for (const book of await dirs(mangaDir)) {
+    if (!parseKey(book)) continue;
+    for (const chapter of await dirs(path.join(mangaDir, book))) {
+      const parsed = parseKey(chapter);
+      if (!parsed || parsed.x) continue; // extras and specials have no chapter number to match
+      const file = path.join(mangaDir, book, chapter, 'chapter.json');
+      let meta;
+      try { meta = JSON.parse(await fs.readFile(file, 'utf8')); } catch { continue; }
+      out.push({ key: chapterKey(parsed.num), book, file, meta });
+    }
+  }
+  return out;
+}
+
+// Look up, decide and (with apply) write x-dates.release for one manga folder.
+async function refreshMangaReleaseDates(mangaDir, sources, { apply = false, pages = Infinity, maxAgeMs } = {}) {
+  const meta = JSON.parse(await fs.readFile(path.join(mangaDir, 'manga.json'), 'utf8').catch(() => '{}'));
+  const versions = await chapterVersions(mangaDir);
+  const summary = { manga: path.basename(mangaDir), series: null, chapters: new Set(versions.map(v => v.key)).size, official: 0, accepted: 0, written: 0, cleared: 0, rejected: null, ambiguous: false };
+  // Seasons that restart at chapter 1 cannot be matched to MangaUpdates' running numbers.
+  const books = new Map();
+  for (const version of versions) books.set(version.key, (books.get(version.key) || new Set()).add(version.book));
+  if ([...books.values()].some(set => set.size > 1)) { summary.ambiguous = true; return summary; }
+  const series = await sources.findSeries(meta);
+  if (!series) return summary;
+  summary.series = series;
+  const run = await sources.originalRun(meta['x-anilist']?.id || meta['x-scraper']?.ids?.anilist);
+  const evidence = new Map();
+  for (const version of versions) {
+    const time = Date.parse(version.meta['x-dates']?.source?.at);
+    if (version.key && Number.isFinite(time) && time < (evidence.get(version.key) ?? Infinity)) evidence.set(version.key, time);
+  }
+  const { accepted, rejected, officialCount } = officialDates(await sources.releases(series.id, { pages, ...(maxAgeMs ? { maxAgeMs } : {}) }), run, evidence);
+  summary.official = officialCount; summary.rejected = rejected;
+  const have = new Set(versions.map(v => v.key));
+  summary.accepted = [...accepted.keys()].filter(key => have.has(key)).length;
+  for (const version of versions) {
+    const dates = version.meta['x-dates'] || { version: 1 }, next = accepted.get(version.key);
+    const prior = dates.release;
+    let changed = false;
+    if (next && JSON.stringify(prior) !== JSON.stringify(next)) { dates.release = next; changed = true; summary.written++; }
+    // A date this job wrote earlier and no longer accepts is withdrawn; other release dates stay.
+    else if (!next && prior && /^MangaUpdates\b/.test(prior.source || '') && pages === Infinity) { delete dates.release; changed = true; summary.cleared++; }
+    if (changed && apply) await writeJsonAtomic(version.file, { ...version.meta, 'x-dates': dates });
+  }
+  return summary;
+}
+
+module.exports = { ReleaseDateSources, officialDates, refreshMangaReleaseDates, chapterKey, isOfficial, EARLY_TOLERANCE_DAYS };

scraper/service.js

@@ -13,6 +13,7 @@ const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, summarizeInv
 const { inspectLegacyWork, promoteLegacyWork } = require('./lib/legacy-publication');
 const { classifyFailure, safeMessage, progressKey } = require('./lib/failures');
 const { writeJsonAtomic } = require('./lib/json-file');
+const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates');
 
 const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs'];
 const FAST_RATE_SETTINGS = {
@@ -353,12 +354,36 @@ class ScraperService {
     const delay = Math.max(1000, next.getTime() - Date.now());
     this.discoveryTimer = setTimeout(() => {
       this.discoveryTimer = null;
-      this.enqueueDiscoveryChecks('schedule').catch(error => this.event('error', `Nightly discovery failed: ${error.message}`))
+      this.refreshReleaseDates('schedule').catch(error => this.event('error', `Nightly release dates failed: ${error.message}`))
+        .then(() => this.enqueueDiscoveryChecks('schedule'))
+        .catch(error => this.event('error', `Nightly discovery failed: ${error.message}`))
         .finally(() => { if (!this.stopping) this.scheduleDiscovery(); });
     }, delay);
     this.discoveryTimer.unref();
   }
 
+  // Nightly, before the update checks: official release dates of ongoing series from their
+  // newest MangaUpdates releases (BL-0048). Runs only while no download is active, so the
+  // scraper stays the single writer of chapter.json.
+  async refreshReleaseDates(reason = 'manual') {
+    if (this.active.size) { await this.event('info', 'Release dates skipped: a download is running', { reason }); return null; }
+    const inventory = await this.libraryInventory(true);
+    const sources = new ReleaseDateSources({ cacheDir: path.join(this.dataRoot, 'release-dates-cache') });
+    let checked = 0, written = 0;
+    for (const manga of inventory.mangas) {
+      if (this.stopping || this.active.size) break;
+      if (manga.meta?.status !== 'ongoing') continue;
+      try {
+        const result = await refreshMangaReleaseDates(manga.path, sources, { apply: true, pages: 1, maxAgeMs: 20 * 3600e3 });
+        checked++; written += result.written;
+      } catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); }
+    }
+    // The reader reconciles fully when the revision rises without a visible folder change.
+    if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
+    await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written });
+    return { checked, written };
+  }
+
   pendingSeries(source, seriesId) {
     return this.state.queue.some(item => item.source === source && String(item.seriesId) === String(seriesId)
       && ['queued', 'downloading', 'discovering', 'ready', 'error', 'retry_wait', 'quarantined', 'unavailable'].includes(item.status));

scraper/test/release-dates.test.js

@@ -0,0 +1,93 @@
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs').promises;
+const os = require('node:os');
+const path = require('node:path');
+const { officialDates, refreshMangaReleaseDates, chapterKey } = require('../metadata/release-dates');
+
+const DAY = 864e5;
+const run = (start, end = null) => ({ start: Date.parse(start), end: end ? Date.parse(end) : null });
+const viz = (chapter, date) => ({ chapter, date, groups: ['Viz', 'MANGA Plus'] });
+const scan = (chapter, date) => ({ chapter, date, groups: ['Some Scans'] });
+
+test('chapter keys only match single numbered chapters', () => {
+  assert.equal(chapterKey('001'), '1');
+  assert.equal(chapterKey('10.50'), '10.5');
+  assert.equal(chapterKey('1-3'), null);
+  assert.equal(chapterKey('7a'), null);
+});
+
+test('simulpub dates are accepted, re-releases after the run are not', () => {
+  const ongoing = officialDates([viz('1', '2023-09-17'), viz('2', '2023-09-24'), scan('2', '2023-09-23')], run('2023-09-18'));
+  assert.deepEqual([...ongoing.accepted.keys()].sort(), ['1', '2'], 'scans a day early do not disqualify');
+  assert.equal(ongoing.accepted.get('1').at, '2023-09-17');
+  assert.equal(ongoing.accepted.get('1').source, 'MangaUpdates · Viz, MANGA Plus');
+  const ended = officialDates([viz('700', '2020-12-20')], run('1999-09-21', '2014-11-10'));
+  assert.equal(ended.accepted.size, 0);
+  assert.equal(ended.rejected.outsideRun, 1);
+  assert.equal(officialDates([viz('1', '2020-01-01')], null).rejected.noRun, 1);
+});
+
+test('an earlier scanlation or source upload means the official date is a re-release', () => {
+  const scanned = officialDates([viz('5', '2021-06-01'), scan('5', '2019-01-01')], run('2018-01-01'));
+  assert.equal(scanned.rejected.earlier, 1);
+  const uploaded = officialDates([viz('5', '2021-06-01')], run('2018-01-01'), new Map([['5', Date.parse('2021-05-01')]]));
+  assert.equal(uploaded.rejected.earlier, 1);
+  const leak = officialDates([viz('5', '2021-06-04')], run('2018-01-01'), new Map([['5', Date.parse('2021-06-04') - 2 * DAY]]));
+  assert.equal(leak.accepted.size, 1, 'an upload two days early is a leak, not an older release');
+});
+
+test('back-catalog chapters dated after newer chapters are out of order', () => {
+  const result = officialDates([
+    viz('8', '2019-02-06'), viz('33', '2019-07-31'), // weekly back-catalog re-release
+    viz('928', '2018-12-21'), viz('930', '2019-01-20'), viz('931', '2019-02-03'), viz('1196', '2026-10-18'),
+    viz('929', '2019-01-21'), // a day after 930: a double release is still in order
+  ], run('1997-07-22'));
+  assert.deepEqual([...result.accepted.keys()].sort((a, b) => a - b), ['928', '929', '930', '931', '1196']);
+  assert.equal(result.rejected.outOfOrder, 2);
+});
+
+async function library(chapters) {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'release-dates-'));
+  const manga = path.join(root, 'Kagurabachi');
+  await fs.mkdir(manga, { recursive: true });
+  await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Kagurabachi', year: 2023, 'x-anilist': { id: 169355 } }));
+  for (const [folder, meta] of Object.entries(chapters)) {
+    await fs.mkdir(path.join(manga, folder), { recursive: true });
+    await fs.writeFile(path.join(manga, folder, 'chapter.json'), JSON.stringify(meta));
+  }
+  return manga;
+}
+const fakeSources = releases => ({
+  async findSeries(meta) { return meta.title === 'Kagurabachi' ? { id: '60441951323', title: 'Kagurabachi' } : null; },
+  async originalRun() { return run('2023-09-18'); },
+  async releases() { return releases; },
+});
+
+test('refresh writes x-dates.release, keeps other dates and withdraws its own stale dates', async () => {
+  const manga = await library({
+    'v01/c0001 [WeebCentral]': { title: '', added: '2026-10-01T00:00:00Z', 'x-dates': { version: 1, source: { at: '2024-09-07T10:00:00Z', name: 'WeebCentral', batch: true } } },
+    'v01/c0002 [WeebCentral]': { title: '', 'x-dates': { version: 1, release: { at: '2020-01-01', precision: 'day', source: 'MangaUpdates · Viz' } } },
+    'v01/c0003 [WeebCentral]': { title: '', 'x-dates': { version: 1, release: { at: '2023-10-01', precision: 'day', source: 'Hand-entered' } } },
+  });
+  const dry = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')]));
+  assert.equal(dry.written, 1);
+  assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0001 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release, undefined, 'dry run writes nothing');
+  const applied = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')]), { apply: true });
+  assert.deepEqual([applied.written, applied.cleared], [1, 1]);
+  const one = JSON.parse(await fs.readFile(path.join(manga, 'v01/c0001 [WeebCentral]/chapter.json'), 'utf8'));
+  assert.deepEqual(one['x-dates'].release, { at: '2023-09-17', precision: 'day', source: 'MangaUpdates · Viz, MANGA Plus' });
+  assert.equal(one['x-dates'].source.batch, true);
+  assert.equal(one.added, '2026-10-01T00:00:00Z');
+  assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0002 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release, undefined);
+  assert.equal(JSON.parse(await fs.readFile(path.join(manga, 'v01/c0003 [WeebCentral]/chapter.json'), 'utf8'))['x-dates'].release.source, 'Hand-entered');
+});
+
+test('seasons that restart their numbering are skipped', async () => {
+  const manga = await library({ 'v01 Season 1/c0001': { title: '' }, 'v02 Season 2/c0001': { title: '' } });
+  const result = await refreshMangaReleaseDates(manga, fakeSources([viz('1', '2023-09-17')]), { apply: true });
+  assert.equal(result.ambiguous, true);
+  assert.equal(result.written, 0);
+});