MangaCircus/scraper/metadata/release-dates.js

'use strict';
// Official chapter release dates (BL-0048) from MangaUpdates publisher releases.
// MangaUpdates also lists English digital re-releases of old chapters as "official"
// (Naruto ch. 700 shows 2020; it came out in Japan in 2014). A date is accepted only when
//   1. it lies inside the work's original run (AniList start - 7 days ... end + 30 days), and
//   2. nothing shows the chapter earlier: no scanlation release in MangaUpdates and no
//      source upload (bulk or not) more than EARLY_TOLERANCE_DAYS before it, and
//   3. it fits the release order (see officialDates).
// Simulpub releases pass all three; back-catalog re-releases fail at least one.
const fs = require('node:fs').promises;
const path = require('node:path');
const crypto = require('node:crypto');
const { parseKey } = require('../../chapter-identity');
const { writeJsonAtomic } = require('../lib/json-file');

const DAY = 864e5;
const EARLY_TOLERANCE_DAYS = 4; // early scans and source uploads of weekly series arrive 1-3 days before the official date
const ORDER_TOLERANCE_DAYS = 2; // double releases and day-granular dates may put a lower chapter a day or two later
const OFFICIAL = new Set(['viz', 'viz media', 'manga plus', 'mangaplus', 'shueisha', 'kodansha', 'kodansha usa', 'k manga',
  'yen press', 'seven seas', 'square enix', 'square enix manga', 'manga up!', 'comikey', 'azuki', 'tappytoon', 'webtoon',
  'line webtoon', 'webtoons', 'tapas', 'pocket comics', 'manta', 'lezhin', 'lezhin comics', 'j-novel club', 'dark horse',
  'vertical', 'denpa', 'tokyopop', 'ize press', 'one peace books', 'inky pen', 'kakaopage', 'kakao webtoon', 'naver webtoon']);
const UA = 'MangaCircus/1.0 (self-hosted manga reader; chapter release dates)';
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
const norm = value => String(value || '').toLowerCase().normalize('NFKD').replace(/[^a-z0-9]+/g, '');
// "001", "1.50" and "1" are the same chapter; ranges, volumes and letters are not single chapters.
const chapterKey = value => (/^\s*\d+(?:\.\d+)?\s*$/.test(String(value ?? '')) ? String(Number(value)) : null);
const isOfficial = groups => groups.some(name => OFFICIAL.has(String(name).toLowerCase().trim()));

async function cached(cacheDir, key, maxAgeMs, load) {
  const file = cacheDir && path.join(cacheDir, crypto.createHash('sha256').update(key).digest('hex').slice(0, 32) + '.json');
  if (file) {
    try {
      const stat = await fs.stat(file);
      if (Date.now() - stat.mtimeMs < maxAgeMs) return JSON.parse(await fs.readFile(file, 'utf8'));
    } catch { /* not cached */ }
  }
  const value = await load();
  if (file) { await fs.mkdir(cacheDir, { recursive: true }); await writeJsonAtomic(file, value); }
  return value;
}

class ReleaseDateSources {
  constructor(options = {}) {
    this.cacheDir = options.cacheDir || null;
    this.minGapMs = options.minGapMs ?? 1100;
    this.fetch = options.fetch || globalThis.fetch;
    this.last = 0;
  }
  async request(url, body) {
    const wait = this.last + this.minGapMs - Date.now();
    if (wait > 0) await sleep(wait);
    this.last = Date.now();
    const response = await this.fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', 'user-agent': UA },
      body: JSON.stringify(body), signal: AbortSignal.timeout(30000) });
    if (!response.ok) throw new Error(`${new URL(url).hostname} HTTP ${response.status}`);
    return response.json();
  }
  // MangaUpdates series whose matched name equals one of the manga's titles and whose year fits.
  async findSeries(meta) {
    const titles = [meta.title, ...(Array.isArray(meta.altTitles) ? meta.altTitles : [])].map(norm).filter(Boolean);
    const year = Number(meta.year) || null;
    const result = await cached(this.cacheDir, `mu-search:${meta.title}`, 30 * DAY,
      () => this.request('https://api.mangaupdates.com/v1/series/search', { search: String(meta.title || ''), perpage: 10 }));
    for (const hit of result.results || []) {
      const record = hit.record || {}, hitYear = Number(record.year) || null;
      if (year && hitYear && Math.abs(hitYear - year) > 1) continue;
      if ([record.title, hit.hit_title].map(norm).some(name => name && titles.includes(name))) {
        return { id: String(record.series_id), title: record.title, url: record.url || null };
      }
    }
    return null;
  }
  async releases(seriesId, { pages = Infinity, maxAgeMs = 7 * DAY } = {}) {
    return cached(this.cacheDir, `mu-releases:${seriesId}:${pages}`, maxAgeMs, async () => {
      const rows = [];
      for (let page = 1; page <= pages; page++) {
        const result = await this.request('https://api.mangaupdates.com/v1/releases/search',
          { search: String(seriesId), search_type: 'series', perpage: 100, page, orderby: 'date' });
        for (const { record } of result.results || []) {
          if (record?.release_date) rows.push({ chapter: record.chapter, date: record.release_date, groups: (record.groups || []).map(g => g.name) });
        }
        if (!result.results?.length || page * 100 >= (result.total_hits || 0)) break;
      }
      return rows;
    });
  }
  async originalRun(anilistId) {
    if (!anilistId) return null;
    const result = await cached(this.cacheDir, `anilist-run:${anilistId}`, 30 * DAY, () => this.request('https://graphql.anilist.co', {
      query: 'query ($id: Int) { Media(id: $id, type: MANGA) { status startDate { year month day } endDate { year month day } } }',
      variables: { id: Number(anilistId) },
    }));
    const media = result.data?.Media;
    if (!media) return null;
    const date = (d, end) => (d?.year ? Date.UTC(d.year, (d.month || (end ? 12 : 1)) - 1, d.day || (end ? 28 : 1)) : null);
    return { status: media.status, start: date(media.startDate, false), end: media.status === 'RELEASING' ? null : date(media.endDate, true) };
  }
}

// Pure decision: official dates per chapter key, with the reasons others were rejected.
// evidence: Map chapterKey -> earliest other time (ms) the library knows for that chapter.
function officialDates(releases, run, evidence = new Map()) {
  const official = new Map(), scans = new Map();
  for (const row of releases) {
    const key = chapterKey(row.chapter), time = Date.parse(row.date);
    if (!key || !Number.isFinite(time)) continue;
    const target = isOfficial(row.groups) ? official : scans;
    const prior = target.get(key);
    if (!prior || time < prior.time) target.set(key, { time, date: String(row.date).slice(0, 10), groups: row.groups });
  }
  const accepted = new Map(), rejected = { outsideRun: 0, earlier: 0, outOfOrder: 0, noRun: 0 };
  // 3. Walking down from the newest chapter, release dates never rise again: a lower chapter
  //    dated after a higher one is a back-catalog re-release (One Piece ch. 8 on 2019-02-06).
  let ceiling = Infinity;
  for (const [key, item] of [...official].sort((a, b) => Number(b[0]) - Number(a[0]))) {
    if (!run || !run.start) { rejected.noRun++; continue; }
    if (item.time < run.start - 7 * DAY || (run.end && item.time > run.end + 30 * DAY)) { rejected.outsideRun++; continue; }
    const earliest = Math.min(scans.get(key)?.time ?? Infinity, evidence.get(key) ?? Infinity);
    if (earliest < item.time - EARLY_TOLERANCE_DAYS * DAY) { rejected.earlier++; continue; }
    if (item.time > ceiling + ORDER_TOLERANCE_DAYS * DAY) { rejected.outOfOrder++; continue; }
    ceiling = Math.min(ceiling, item.time);
    accepted.set(key, { at: item.date, precision: 'day', source: `MangaUpdates · ${item.groups.join(', ')}` });
  }
  return { accepted, rejected, officialCount: official.size };
}

// Chapter version folders of one manga with their numeric chapter key.
async function chapterVersions(mangaDir) {
  const out = [];
  const dirs = async dir => (await fs.readdir(dir, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name);
  for (const book of await dirs(mangaDir)) {
    if (!parseKey(book)) continue;
    for (const chapter of await dirs(path.join(mangaDir, book))) {
      const parsed = parseKey(chapter);
      if (!parsed || parsed.x) continue; // extras and specials have no chapter number to match
      const file = path.join(mangaDir, book, chapter, 'chapter.json');
      let meta;
      try { meta = JSON.parse(await fs.readFile(file, 'utf8')); } catch { continue; }
      out.push({ key: chapterKey(parsed.num), book, file, meta });
    }
  }
  return out;
}

// Look up, decide and (with apply) write x-dates.release for one manga folder.
async function refreshMangaReleaseDates(mangaDir, sources, { apply = false, pages = Infinity, maxAgeMs } = {}) {
  const meta = JSON.parse(await fs.readFile(path.join(mangaDir, 'manga.json'), 'utf8').catch(() => '{}'));
  const versions = await chapterVersions(mangaDir);
  const summary = { manga: path.basename(mangaDir), series: null, chapters: new Set(versions.map(v => v.key)).size, official: 0, accepted: 0, written: 0, cleared: 0, rejected: null, ambiguous: false };
  // Seasons that restart at chapter 1 cannot be matched to MangaUpdates' running numbers.
  const books = new Map();
  for (const version of versions) books.set(version.key, (books.get(version.key) || new Set()).add(version.book));
  if ([...books.values()].some(set => set.size > 1)) { summary.ambiguous = true; return summary; }
  const series = await sources.findSeries(meta);
  if (!series) return summary;
  summary.series = series;
  const run = await sources.originalRun(meta['x-anilist']?.id || meta['x-scraper']?.ids?.anilist);
  const evidence = new Map();
  for (const version of versions) {
    const time = Date.parse(version.meta['x-dates']?.source?.at);
    if (version.key && Number.isFinite(time) && time < (evidence.get(version.key) ?? Infinity)) evidence.set(version.key, time);
  }
  const { accepted, rejected, officialCount } = officialDates(await sources.releases(series.id, { pages, ...(maxAgeMs ? { maxAgeMs } : {}) }), run, evidence);
  summary.official = officialCount; summary.rejected = rejected;
  const have = new Set(versions.map(v => v.key));
  summary.accepted = [...accepted.keys()].filter(key => have.has(key)).length;
  for (const version of versions) {
    const dates = version.meta['x-dates'] || { version: 1 }, next = accepted.get(version.key);
    const prior = dates.release;
    let changed = false;
    if (next && JSON.stringify(prior) !== JSON.stringify(next)) { dates.release = next; changed = true; summary.written++; }
    // A date this job wrote earlier and no longer accepts is withdrawn; other release dates stay.
    else if (!next && prior && /^MangaUpdates\b/.test(prior.source || '') && pages === Infinity) { delete dates.release; changed = true; summary.cleared++; }
    if (changed && apply) await writeJsonAtomic(version.file, { ...version.meta, 'x-dates': dates });
  }
  return summary;
}

module.exports = { ReleaseDateSources, officialDates, refreshMangaReleaseDates, chapterKey, isOfficial, EARLY_TOLERANCE_DAYS };