Chapter dates: read date setting and tiered release dates (BL-0047)
- Chapter rows show when you first finished a chapter (setting: date + hour, date, off) from a new durable progress doneAt; older reads fall back to the reading clock's earliest remembered completion. - chapter.json keeps added (arrival in this library, now kept on replace) plus x-dates.source (source upload, bulk imports flagged), x-dates.release (official, for a later metadata job) and x-dates.ecosystem (first entry into any Manga Circus library; carried over peering, earliest wins, clamped). - Rows show one release date by tier: official > source upload > ecosystem > library, with all known dates in the tooltip (setting to hide). - Fixes: WeebCentral dates came from the chapter page, which has none (every chapter got the download time); they now come from the full chapter list. MangaDex no longer writes a group's publishAt into added. Peering sends dates. - tools/backfill-chapter-dates.js migrates an existing library (dry run default). - Coverage dry run on the server: docs/chapter-dates-coverage-2026-10-24.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b2bb34174c
commit
e0dacd81c6
21 files changed
+470
-32
No files matched your search
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// One-time migration to chapter dates (BL-0047). Dry run by default; apply only with the
|
||||
// scraper stopped (--apply --writers-stopped), then run a deep "Rescan" in Settings.
|
||||
// - WeebCentral: x-dates.source from the series' full chapter list (one request per series).
|
||||
// - MangaDex: chapters published before x-dates stored the group's publishAt in "added";
|
||||
// it moves to x-dates.source and "added" becomes the folder's arrival time.
|
||||
// - Every chapter: x-dates.ecosystem = "added" unless an earlier one is already known.
|
||||
const fs = require('node:fs').promises;
|
||||
const path = require('node:path');
|
||||
const { batchFlags, sourceDate, publishedDates } = require('../chapter-dates');
|
||||
const weebcentral = require('../scraper/plugins/weebcentral');
|
||||
const { writeJsonAtomic } = require('../scraper/lib/publisher');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const apply = args.includes('--apply');
|
||||
const only = args.find(arg => arg.startsWith('--manga='))?.slice('--manga='.length);
|
||||
const offline = args.includes('--offline');
|
||||
const root = args.find(arg => !arg.startsWith('--'));
|
||||
if (!root || args.some(arg => arg.startsWith('--') && !['--apply', '--writers-stopped', '--offline'].includes(arg) && !arg.startsWith('--manga='))
|
||||
|| (apply && !args.includes('--writers-stopped'))) {
|
||||
console.error('Usage: node tools/backfill-chapter-dates.js <library> [--manga=<folder>] [--offline] [--apply --writers-stopped]');
|
||||
process.exit(2);
|
||||
}
|
||||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130 Safari/537.36';
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const readJson = async file => { try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; } };
|
||||
const dirs = async dir => (await fs.readdir(dir, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name).sort();
|
||||
|
||||
async function weebCentralUploads(seriesId) {
|
||||
const url = `https://weebcentral.com/series/${seriesId}/full-chapter-list`;
|
||||
const response = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, signal: AbortSignal.timeout(30000) });
|
||||
if (!response.ok) throw new Error(`WeebCentral HTTP ${response.status}`);
|
||||
const { chapters } = weebcentral.catalog({ html: await response.text(), url });
|
||||
return new Map(chapters.filter(c => c.uploadedAt).map(c => [c.id, sourceDate(c.uploadedAt, 'WeebCentral', c.uploadBatch)]));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const totals = { chapters: 0, changed: 0, alreadyDated: 0, weebSource: 0, weebBatch: 0, mangaDexMoved: 0, missing: 0 };
|
||||
for (const manga of await dirs(root)) {
|
||||
if (only && manga !== only) continue;
|
||||
const mangaDir = path.join(root, manga), meta = (await readJson(path.join(mangaDir, 'manga.json'))) || {};
|
||||
const versions = [];
|
||||
for (const book of await dirs(mangaDir)) {
|
||||
for (const chapter of await dirs(path.join(mangaDir, book))) {
|
||||
const dir = path.join(mangaDir, book, chapter), file = path.join(dir, 'chapter.json'), chapterMeta = await readJson(file);
|
||||
if (chapterMeta) versions.push({ dir, file, meta: chapterMeta, mtime: (await fs.stat(dir)).mtimeMs });
|
||||
}
|
||||
}
|
||||
let uploads = new Map();
|
||||
const seriesId = meta['x-scraper']?.ids?.weebcentral;
|
||||
if (seriesId && !offline && versions.some(v => /weebcentral\.com\/chapters\//.test(v.meta.source?.url || '') && !v.meta['x-dates'])) {
|
||||
try { uploads = await weebCentralUploads(seriesId); await sleep(2000); }
|
||||
catch (error) { console.warn(`[backfill] ${manga}: WeebCentral list failed: ${error.message}`); }
|
||||
}
|
||||
// Legacy MangaDex "added" values are group upload times; flag bulk uploads among them.
|
||||
const legacyDex = versions.filter(v => !v.meta['x-dates'] && /^MangaDex\b/.test(v.meta.source?.name || '') && Date.parse(v.meta.added));
|
||||
const dexBatch = new Map(batchFlags(legacyDex.map(v => v.meta.added)).map((flag, i) => [legacyDex[i], flag]));
|
||||
let changed = 0;
|
||||
for (const version of versions) {
|
||||
totals.chapters++;
|
||||
if (version.meta['x-dates']) { totals.alreadyDated++; continue; }
|
||||
const prior = { ...version.meta }, incoming = {};
|
||||
const weebId = /weebcentral\.com\/chapters\/([0-9A-Z]{26})/.exec(prior.source?.url || '')?.[1];
|
||||
if (weebId && uploads.has(weebId)) {
|
||||
incoming.source = uploads.get(weebId);
|
||||
totals[incoming.source.batch ? 'weebBatch' : 'weebSource']++;
|
||||
} else if (dexBatch.has(version)) {
|
||||
incoming.source = sourceDate(prior.added, prior.source.name, dexBatch.get(version));
|
||||
prior.added = new Date(version.mtime).toISOString();
|
||||
totals.mangaDexMoved++;
|
||||
} else totals.missing++;
|
||||
const next = { ...prior, ...publishedDates(prior, incoming, version.mtime) };
|
||||
changed++; totals.changed++;
|
||||
if (apply) await writeJsonAtomic(version.file, next);
|
||||
}
|
||||
if (changed) console.log(`[backfill] ${manga}: ${changed} of ${versions.length} chapter versions ${apply ? 'updated' : 'would change'}`);
|
||||
}
|
||||
console.log(JSON.stringify({ apply, ...totals }));
|
||||
if (apply) console.log('Run a deep "Rescan" in Settings so the reader picks up the edited chapter.json files.');
|
||||
}
|
||||
|
||||
main().catch(error => { console.error(error); process.exit(1); });
|
||||
Reference in new issue
Block a user