#!/usr/bin/env node 'use strict'; // One-time migration to chapter dates (BL-0047). Dry run by default; apply only with the // scraper stopped (--apply --writers-stopped), then run a deep "Rescan" in Settings. // - WeebCentral: x-dates.source from the series' full chapter list (one request per series). // - MangaDex: chapters published before x-dates stored the group's publishAt in "added"; // it moves to x-dates.source and "added" becomes the folder's arrival time. // - Every chapter: x-dates.ecosystem = "added" unless an earlier one is already known. const fs = require('node:fs').promises; const path = require('node:path'); const { batchFlags, sourceDate, publishedDates } = require('../chapter-dates'); const weebcentral = require('../scraper/plugins/weebcentral'); const { writeJsonAtomic } = require('../scraper/lib/publisher'); const args = process.argv.slice(2); const apply = args.includes('--apply'); const only = args.find(arg => arg.startsWith('--manga='))?.slice('--manga='.length); const offline = args.includes('--offline'); const root = args.find(arg => !arg.startsWith('--')); if (!root || args.some(arg => arg.startsWith('--') && !['--apply', '--writers-stopped', '--offline'].includes(arg) && !arg.startsWith('--manga=')) || (apply && !args.includes('--writers-stopped'))) { console.error('Usage: node tools/backfill-chapter-dates.js [--manga=] [--offline] [--apply --writers-stopped]'); process.exit(2); } const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130 Safari/537.36'; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); const readJson = async file => { try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; } }; const dirs = async dir => (await fs.readdir(dir, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name).sort(); async function weebCentralUploads(seriesId) { const url = `https://weebcentral.com/series/${seriesId}/full-chapter-list`; const response = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, signal: AbortSignal.timeout(30000) }); if (!response.ok) throw new Error(`WeebCentral HTTP ${response.status}`); const { chapters } = weebcentral.catalog({ html: await response.text(), url }); return new Map(chapters.filter(c => c.uploadedAt).map(c => [c.id, sourceDate(c.uploadedAt, 'WeebCentral', c.uploadBatch)])); } async function main() { const totals = { chapters: 0, changed: 0, alreadyDated: 0, weebSource: 0, weebBatch: 0, mangaDexMoved: 0, missing: 0 }; for (const manga of await dirs(root)) { if (only && manga !== only) continue; const mangaDir = path.join(root, manga), meta = (await readJson(path.join(mangaDir, 'manga.json'))) || {}; const versions = []; for (const book of await dirs(mangaDir)) { for (const chapter of await dirs(path.join(mangaDir, book))) { const dir = path.join(mangaDir, book, chapter), file = path.join(dir, 'chapter.json'), chapterMeta = await readJson(file); if (chapterMeta) versions.push({ dir, file, meta: chapterMeta, mtime: (await fs.stat(dir)).mtimeMs }); } } let uploads = new Map(); const seriesId = meta['x-scraper']?.ids?.weebcentral; if (seriesId && !offline && versions.some(v => /weebcentral\.com\/chapters\//.test(v.meta.source?.url || '') && !v.meta['x-dates'])) { try { uploads = await weebCentralUploads(seriesId); await sleep(2000); } catch (error) { console.warn(`[backfill] ${manga}: WeebCentral list failed: ${error.message}`); } } // Legacy MangaDex "added" values are group upload times; flag bulk uploads among them. const legacyDex = versions.filter(v => !v.meta['x-dates'] && /^MangaDex\b/.test(v.meta.source?.name || '') && Date.parse(v.meta.added)); const dexBatch = new Map(batchFlags(legacyDex.map(v => v.meta.added)).map((flag, i) => [legacyDex[i], flag])); let changed = 0; for (const version of versions) { totals.chapters++; if (version.meta['x-dates']) { totals.alreadyDated++; continue; } const prior = { ...version.meta }, incoming = {}; const weebId = /weebcentral\.com\/chapters\/([0-9A-Z]{26})/.exec(prior.source?.url || '')?.[1]; if (weebId && uploads.has(weebId)) { incoming.source = uploads.get(weebId); totals[incoming.source.batch ? 'weebBatch' : 'weebSource']++; } else if (dexBatch.has(version)) { incoming.source = sourceDate(prior.added, prior.source.name, dexBatch.get(version)); prior.added = new Date(version.mtime).toISOString(); totals.mangaDexMoved++; } else totals.missing++; const next = { ...prior, ...publishedDates(prior, incoming, version.mtime) }; changed++; totals.changed++; if (apply) await writeJsonAtomic(version.file, next); } if (changed) console.log(`[backfill] ${manga}: ${changed} of ${versions.length} chapter versions ${apply ? 'updated' : 'would change'}`); } console.log(JSON.stringify({ apply, ...totals })); if (apply) console.log('Run a deep "Rescan" in Settings so the reader picks up the edited chapter.json files.'); } main().catch(error => { console.error(error); process.exit(1); });