diff --git a/.dockerignore b/.dockerignore index cf9ae11..6775392 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,7 @@ !library.js !reading-direction.js !chapter-identity.js +!chapter-dates.js !catalog.js !backup.js !reading-analytics.js diff --git a/Dockerfile b/Dockerfile index fd7b4fd..374640a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apk add --no-cache tzdata openssl WORKDIR /app ARG VCS_REF=development LABEL org.opencontainers.image.title="MangaCircus" org.opencontainers.image.revision=$VCS_REF -COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js reading-access.js catalog.js backup.js ./ +COPY package.json server.js library.js reading-direction.js chapter-identity.js chapter-dates.js reading-analytics.js reading-access.js catalog.js backup.js ./ COPY public ./public COPY greatest ./greatest COPY discovery ./discovery diff --git a/chapter-dates.js b/chapter-dates.js new file mode 100644 index 0000000..078c7bb --- /dev/null +++ b/chapter-dates.js @@ -0,0 +1,109 @@ +'use strict'; +// Chapter dates (BL-0047). chapter.json keeps three independent dates: +// added when the chapter entered this library (kept on replace) +// x-dates.source when the source site uploaded it; batch=true marks a bulk import, not a release +// x-dates.release the official public release (filled by a metadata job; precision "day") +// x-dates.ecosystem when it first entered any Manga Circus library; travels over peering, earliest wins +// The reader shows one release date per chapter by tier: official > source (not batch) > ecosystem > library. + +const BATCH_SIZE = 3; // this many uploads ... +const BATCH_GAP_MS = 10 * 60e3; // ... each within 10 minutes of the next form a bulk import + +const iso = value => { + const time = typeof value === 'number' ? value : Date.parse(value); + return Number.isFinite(time) && time > 0 ? new Date(time).toISOString() : null; +}; +const text = (value, max = 120) => (typeof value === 'string' ? value.trim().slice(0, max) : ''); + +// Returns one flag per item: true when its upload time belongs to a bulk import, null without a time. +function batchFlags(times) { + const order = times.map((value, index) => ({ index, time: Date.parse(value) })) + .filter(item => Number.isFinite(item.time)).sort((a, b) => a.time - b.time); + const flags = times.map(() => null); + for (let start = 0; start < order.length;) { + let end = start + 1; + while (end < order.length && order[end].time - order[end - 1].time <= BATCH_GAP_MS) end++; + for (let i = start; i < end; i++) flags[order[i].index] = end - start >= BATCH_SIZE; + start = end; + } + return flags; +} + +function sourceDate(at, name, batch) { + const value = iso(at); + return value ? { at: value, name: text(name), batch: !!batch } : null; +} + +function cleanSource(value) { + return value && typeof value === 'object' ? sourceDate(value.at, value.name, value.batch) : null; +} +function cleanRelease(value) { + if (!value || typeof value !== 'object' || !iso(value.at)) return null; + const precision = ['day', 'month', 'year'].includes(value.precision) ? value.precision : 'day'; + return { at: String(value.at).slice(0, 30), precision, source: text(value.source) }; +} +function cleanEcosystem(value) { + if (!value || typeof value !== 'object' || !iso(value.at)) return null; + return { at: iso(value.at), via: text(value.via) || 'this server' }; +} + +// chapter.json fields for a (re)published chapter. A prior "added" and an earlier ecosystem +// date always survive; a peer's ecosystem claim may lower the date but never raise it. +function publishedDates(priorMeta = {}, incoming = {}, now = Date.now()) { + const prior = priorMeta['x-dates'] && typeof priorMeta['x-dates'] === 'object' ? priorMeta['x-dates'] : {}; + const added = iso(priorMeta.added) || iso(incoming.added) || iso(now); + const source = cleanSource(incoming.source) || cleanSource(prior.source); + const release = cleanRelease(incoming.release) || cleanRelease(prior.release); + let claim = cleanEcosystem(incoming.ecosystem); + if (claim) { + // Nobody can have had the chapter before the source uploaded it, or in the future. + const floor = source ? Date.parse(source.at) : 0, time = Math.min(Date.parse(claim.at), now); + claim = { ...claim, at: iso(Math.max(time, floor)) }; + } + const candidates = [cleanEcosystem(prior.ecosystem), claim, { at: added, via: 'this server' }].filter(Boolean); + const ecosystem = candidates.reduce((best, item) => (Date.parse(item.at) < Date.parse(best.at) ? item : best)); + return { added, 'x-dates': { version: 1, ...(source ? { source } : {}), ...(release ? { release } : {}), ecosystem } }; +} + +// Dates of one library chapter from its versions ({ added: ms, meta: chapter.json }). +function chapterDates(versions) { + const known = {}; + const keep = (slot, time, value) => { + if (Number.isFinite(time) && (!known[slot] || time < known[slot].time)) known[slot] = { time, ...value }; + }; + for (const version of versions) { + const dates = (version.meta && version.meta['x-dates']) || {}; + const release = cleanRelease(dates.release), source = cleanSource(dates.source), ecosystem = cleanEcosystem(dates.ecosystem); + if (release) keep('official', Date.parse(release.at), { at: release.at, precision: release.precision, name: release.source }); + if (source) keep(source.batch ? 'batch' : 'source', Date.parse(source.at), { at: source.at, name: source.name }); + if (ecosystem) keep('ecosystem', Date.parse(ecosystem.at), { at: ecosystem.at, name: ecosystem.via }); + if (Number.isFinite(version.added)) keep('library', version.added, { at: iso(version.added) }); + } + const tier = ['official', 'source', 'ecosystem', 'library'].find(slot => known[slot]) || null; + const out = {}; + for (const [slot, { time, ...value }] of Object.entries(known)) out[slot] = value; + return { tier, release: tier ? out[tier] : null, known: out }; +} + +// Ecosystem dates offered to peers: the earliest per chapter number, falling back to "added" +// for chapters published before x-dates existed (this server was then the one that had it). +function ecosystemOf(meta, fallbackMs) { + const dates = (meta && meta['x-dates']) || {}; + return cleanEcosystem(dates.ecosystem)?.at || iso(meta && meta.added) || iso(fallbackMs); +} +// What a gateway sends with a chapter: only cleaned, non-identifying date fields. +function peerDates(meta, fallbackMs) { + const dates = (meta && meta['x-dates']) || {}; + const source = cleanSource(dates.source), release = cleanRelease(dates.release), ecosystem = ecosystemOf(meta, fallbackMs); + return { ...(ecosystem ? { ecosystem } : {}), ...(source ? { source } : {}), ...(release ? { release } : {}) }; +} +// Incoming peer dates in publishedDates() shape. +function fromPeer(value, alias) { + if (!value || typeof value !== 'object') return {}; + const ecosystem = iso(value.ecosystem); + return { ...(ecosystem ? { ecosystem: { at: ecosystem, via: `peer ${text(alias, 40)}`.trim() } } : {}), + ...(cleanSource(value.source) ? { source: cleanSource(value.source) } : {}), + ...(cleanRelease(value.release) ? { release: cleanRelease(value.release) } : {}) }; +} + +module.exports = { BATCH_SIZE, BATCH_GAP_MS, batchFlags, sourceDate, publishedDates, chapterDates, ecosystemOf, peerDates, fromPeer }; diff --git a/docs/library-format.md b/docs/library-format.md index 7f097dd..d40d648 100644 --- a/docs/library-format.md +++ b/docs/library-format.md @@ -208,11 +208,39 @@ An AniList refresh with a different ID is stopped for review before chapter publ |-------|------|---------| | `title` | string | Chapter title without "Chapter N" (default: title part of the folder name) | | `source` | `{name, url}` or string | Where this version came from (default: `[source]` in the folder name) | -| `added` | ISO date/time string | Release (or download) date, used for "recently updated" (default: the folder's modification time) | +| `added` | ISO date/time string | When this version entered **this** library, used for "recently updated" (default: the folder's modification time). The scraper keeps it when it replaces a version | +| `x-dates` | object | Other chapter dates, written by the scraper (see below) | | `extra` | boolean | Marks the chapter as extra without moving it | | `priority` | number | Tie-breaker between versions with equal votes. Higher wins (default `0`) | | `direction` | as above | Overrides book/manga direction, e.g. a colour webtoon special inside a printed series | +#### Chapter dates (`x-dates`) + +```json +"added": "2026-10-17T15:04:03.424Z", +"x-dates": { + "version": 1, + "source": { "at": "2026-10-18T15:06:05.348Z", "name": "WeebCentral", "batch": false }, + "release": { "at": "2023-09-17", "precision": "day", "source": "MangaUpdates · Viz" }, + "ecosystem": { "at": "2026-10-17T15:04:03.424Z", "via": "this server" } +} +``` + +- `source`: when the source site uploaded this version (WeebCentral's full chapter list, + MangaDex `publishAt`). `batch: true` means it arrived with 3 or more other uploads + less than 10 minutes apart, which is a site import, not a release. +- `release`: the official public release (`precision` `day`, `month` or `year`). Not + filled yet; a metadata job will add it. +- `ecosystem`: when the chapter first entered any Manga Circus library. Peer transfers + carry it, and the earliest known date always wins. A peer's claim can't be earlier than + the source upload or later than now. + +The chapter list shows one release date per chapter, choosing the first tier that has +a date: `release` → `source` unless `batch` (shown as ≈) → `ecosystem` (◎) → `added` (↓). +Each tier takes the earliest value across the chapter's versions. Existing libraries get +these fields from `node tools/backfill-chapter-dates.js `: a dry run by default, +then `--apply --writers-stopped` and a deep Rescan. + Manual directions have priority at **chapter → book → manga**. A `directionOverride` or a plain `direction` without matching generated provenance is manual. Otherwise, the scanner checks the version's existing image dimensions for coherent tall strips diff --git a/docs/peering.md b/docs/peering.md index 3b8ce9d..1cae165 100644 --- a/docs/peering.md +++ b/docs/peering.md @@ -110,8 +110,15 @@ an admin can distinguish pairing from permission. | `GET /peer/v1/access` | Pairing state, certificate expiry and current grant status/scopes; no catalog | | `GET /peer/v1/catalog?cursor=N` | Up to 50 approved opaque title IDs, safe metadata, a stable work identity and cover availability | | `GET /peer/v1/covers/{mangaId}` | Approved cover image only; bounded to 5 MiB and charged to peer byte quota | -| `GET /peer/v1/manga/{id}?cursor=N` | Up to 100 complete chapter IDs, public metadata and the same stable work identity | -| `GET /peer/v1/manga/{id}/chapters/{id}` | Ordered page IDs, MIME, byte size, SHA-256 and ETag | +| `GET /peer/v1/manga/{id}?cursor=N` | Up to 100 complete chapter IDs, public metadata, chapter `dates` and the same stable work identity | +| `GET /peer/v1/manga/{id}/chapters/{id}` | Ordered page IDs, MIME, byte size, SHA-256 and ETag, plus chapter `dates` | + +Chapter `dates` are `{ ecosystem, source?, release? }`. `ecosystem` is the earliest +date at which any version of that chapter number entered a Manga Circus library known +to the sender. Older chapters fall back to their `added` date. The receiver stores the +date as `x-dates.ecosystem` (`via: "peer "`) and only lets it lower an existing +date, never raise it. See `docs/library-format.md`. Older peers omit `dates`, and the +receiver then uses its own arrival time. | `GET /peer/v1/blobs/{id}` | One image, optional `Range: bytes=N-` and `If-Match`; 206 on resume | Unknown/unshared IDs return 404. Revoked serving returns 403; bad/missing grants diff --git a/peering/adapter.js b/peering/adapter.js index 4214fff..2d3644d 100644 --- a/peering/adapter.js +++ b/peering/adapter.js @@ -12,6 +12,7 @@ const { publishManifest } = require('../scraper/lib/publisher'); const { writeJsonAtomic } = require('../scraper/lib/json-file'); const { sleep } = require('../scraper/lib/delay'); const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity'); +const { fromPeer } = require('../chapter-dates'); const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering')); const store = () => new PeeringState(dataRoot()); @@ -132,6 +133,8 @@ async function run(info, item, options) { } const publication = await publishManifest({ version: 1, series, chapters: [{ number: ref.number, title: ref.title, book: { number: Number(ref.volume) || 1 }, + // The peer's ecosystem date travels along: the chapter entered Circus there, not here. + dates: fromPeer(chapter.dates || ref.dates, peer.alias), source: { name: `Peer ${peer.alias} ${peer.id.slice(1, 7)}`, url: sourceUrl }, pages: pages.map(page => ({ path: page.path })), }] }, { libraryRoot: options.libraryRoot, ...(manga ? { mangaDir: manga.path } : {}) }); diff --git a/peering/gateway.js b/peering/gateway.js index 5ce2417..8ad0a90 100644 --- a/peering/gateway.js +++ b/peering/gateway.js @@ -9,6 +9,7 @@ const crypto = require('node:crypto'); const { once } = require('node:events'); const { PeeringState, fp, hash, requirePeer, grantStatus } = require('./state'); const { parseKey, parseSpecialFolder } = require('../chapter-identity'); +const { peerDates } = require('../chapter-dates'); const { resolveWorkIdentity } = require('../scraper/lib/work-identity'); const DATA = path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering')); @@ -116,12 +117,23 @@ async function mangaIndex(peer, share) { if (!pages.length) continue; const id = `c${opaque(`${share.id}/${bookName}/${chapterName}`)}`; const chapterMeta = await jsonFile(path.join(chapterDir, 'chapter.json')); + const dirStat = await fsp.stat(chapterDir); const item = { id, number: parsed.key, volume: book.key, title: String(chapterMeta.title || parsed.title || '').slice(0, 200), - pageCount: pages.length, chapterDir, pages }; + pageCount: pages.length, chapterDir, pages, dates: peerDates(chapterMeta, dirStat.mtimeMs) }; chapters.push(item); byId.set(id, item); if (chapters.length > 10000) fail(413, 'Shared manga chapter limit exceeded'); } } + // Every version of a chapter number offers the earliest ecosystem date known for it. + const earliest = new Map(); + for (const item of chapters) { + const key = `${item.volume}/${item.number}`, at = item.dates.ecosystem; + if (at && (!earliest.has(key) || Date.parse(at) < Date.parse(earliest.get(key)))) earliest.set(key, at); + } + for (const item of chapters) { + const at = earliest.get(`${item.volume}/${item.number}`); + if (at) item.dates = { ...item.dates, ecosystem: at }; + } const result = { at: Date.now(), mangaDir, meta, chapters, byId }; indexCache.set(key, result); return result; @@ -147,7 +159,7 @@ async function chapterManifest(peer, share, chapterId) { size: stat.size, mtimeMs: stat.mtimeMs, at: Date.now() }); if (pages.length > 500) fail(413, 'Chapter page limit exceeded'); } - return { id: chapter.id, number: chapter.number, volume: chapter.volume, title: chapter.title, pages }; + return { id: chapter.id, number: chapter.number, volume: chapter.volume, title: chapter.title, dates: chapter.dates, pages }; } async function streamBlob(req, res, peer, id) { const item = blobCache.get(`${peer.id}:${id}`); @@ -248,7 +260,7 @@ const server = https.createServer({ key: identity.key, cert: identity.cert, requ return json(res, 200, { id: share.id, title: String(index.meta.title || share.mangaId).slice(0, 200), workIdentity: resolveWorkIdentity(index.meta, share.mangaId, identity.id), ...publicMetadata(index.meta), coverAvailable: !!coverFile(index.mangaDir), - chapters: index.chapters.slice(cursor, cursor + 100).map(({ id, number, volume, title, pageCount }) => ({ id, number, volume, title, pageCount })), + chapters: index.chapters.slice(cursor, cursor + 100).map(({ id, number, volume, title, pageCount, dates }) => ({ id, number, volume, title, pageCount, dates })), nextCursor: cursor + 100 < index.chapters.length ? String(cursor + 100) : null }); } const chapterMatch = /^\/peer\/v1\/manga\/(m[0-9a-f]{24})\/chapters\/(c[0-9a-f]{32})$/.exec(url.pathname); diff --git a/peering/test/peering.test.js b/peering/test/peering.test.js index db0430b..b2255b3 100644 --- a/peering/test/peering.test.js +++ b/peering/test/peering.test.js @@ -68,7 +68,8 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t await fsp.mkdir(chapterDir, { recursive: true }); await fsp.writeFile(path.join(mangaDir, 'manga.json'), JSON.stringify({ title: 'Sample Manga', direction: 'ltr' })); await fsp.writeFile(path.join(mangaDir, 'cover.png'), PNG); - await fsp.writeFile(path.join(chapterDir, 'chapter.json'), JSON.stringify({ title: 'The first chapter', source: { name: 'Local' } })); + await fsp.writeFile(path.join(chapterDir, 'chapter.json'), JSON.stringify({ title: 'The first chapter', source: { name: 'Local' }, + added: '2026-10-01T08:00:00.000Z', 'x-dates': { version: 1, source: { at: '2026-09-30T15:00:00.000Z', name: 'WeebCentral', batch: false } } })); await fsp.writeFile(path.join(chapterDir, '0001.png'), PNG); const ga = await gateway(dataA, libA, portA, path.join(root, 'usage-a.json')); let gb = await gateway(dataB, libB, portB, path.join(root, 'usage-b.json')); @@ -134,7 +135,13 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t assert.equal(result.publishedCount, 1); assert.deepEqual(JSON.parse(await fsp.readFile(path.join(libB, 'Sample Manga', 'manga.json'), 'utf8'))['x-work-identity'], remote.workIdentity, 'The stable work identity is stored with the imported peer edition'); - assert.equal(fs.existsSync(path.join(libB, 'Sample Manga', 'v01', 'c0001 The first chapter [Peer ServerA ' + inB.id.slice(1, 7) + ']', '0001.png')), true); + const receivedDir = path.join(libB, 'Sample Manga', 'v01', 'c0001 The first chapter [Peer ServerA ' + inB.id.slice(1, 7) + ']'); + assert.equal(fs.existsSync(path.join(receivedDir, '0001.png')), true); + const received = JSON.parse(await fsp.readFile(path.join(receivedDir, 'chapter.json'), 'utf8')); + assert.deepEqual(received['x-dates'].ecosystem, { at: '2026-10-01T08:00:00.000Z', via: 'peer ServerA' }, + 'The chapter keeps the date it first entered Manga Circus on A, not its arrival on B'); + assert.equal(received['x-dates'].source.at, '2026-09-30T15:00:00.000Z'); + assert.notEqual(received.added, '2026-10-01T08:00:00.000Z', '"added" is the arrival in B\'s own library'); const second = await run(info, { scope: { mode: 'missing' }, existingPolicy: 'skip-any' }, { libraryRoot: libB, settings: { chapterMinMs: 500 }, shouldStop: () => false }); assert.equal(second.publishedCount, 0); diff --git a/public/app.css b/public/app.css index bc315c4..b76f061 100644 --- a/public/app.css +++ b/public/app.css @@ -180,6 +180,11 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; } .ch { display: flex; width: 100%; max-width: 100%; align-items: center; gap: 6px; border-bottom: 1px solid var(--line); white-space: nowrap; } .ch-main { flex: 1 1 auto; min-width: 0; display: grid; grid-template-columns: 76px 52px minmax(70px, 1fr) max-content; gap: 8px; align-items: baseline; padding: 6px 4px; text-decoration: none; } .ch-main.has-time { grid-template-columns: 76px 52px 72px minmax(70px, 1fr) max-content; } +.ch-main.has-date { grid-template-columns: 76px 52px 104px minmax(70px, 1fr) max-content; } +.ch-main.has-time.has-date { grid-template-columns: 76px 52px 104px 72px minmax(70px, 1fr) max-content; } +.ch-date { min-width: 0; overflow: hidden; color: var(--muted); font-size: 13px; font-variant-numeric: tabular-nums; text-overflow: ellipsis; white-space: nowrap; } +.ch-date.tier-official { color: inherit; } +.ch-date.tier-ecosystem, .ch-date.tier-library { opacity: .75; } .ch-main:hover .ch-title { text-decoration: underline; } .ch-label { min-width: 0; overflow: hidden; text-overflow: ellipsis; font-weight: 650; font-variant-numeric: tabular-nums; } .ch-pages { min-width: 0; overflow: hidden; color: var(--muted); font-size: 13px; font-variant-numeric: tabular-nums; text-align: left; white-space: nowrap; } @@ -205,6 +210,10 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; } .ch { flex-wrap: nowrap; gap: 4px; overflow: hidden; } .ch-main { grid-template-columns: 64px 44px minmax(0, 1fr) max-content; gap: 6px; } .ch-main.has-time { grid-template-columns: 64px 44px 62px minmax(0, 1fr) max-content; } + /* Phones: the date takes the page-count column. */ + .ch-main.has-date { grid-template-columns: 64px 64px minmax(0, 1fr) max-content; } + .ch-main.has-time.has-date { grid-template-columns: 64px 64px 62px minmax(0, 1fr) max-content; } + .ch-main.has-date .ch-pages { display: none; } .ch > .btn.small { min-width: 0; padding-left: 5px; padding-right: 5px; } .ch > .state { flex: none; } } diff --git a/public/manga.js b/public/manga.js index a3b6071..e705f19 100644 --- a/public/manga.js +++ b/public/manga.js @@ -213,14 +213,16 @@ const panel = h('div', { class: 'versions', hidden: true }); function draw() { const p = ch.progress, done = p && p.done; - const state = h('button', { class: 'state' + (done ? ' done' : ''), title: done ? 'Mark as unread' : 'Mark as read' }, - done ? '✓ read' : p ? `page ${p.p + 1} / ${p.n}` : 'unread'); + const readAt = done ? readDate(ch.readAt) : null; + const state = h('button', { class: 'state' + (done ? ' done' : ''), title: (readAt ? readAt.title + ' · ' : '') + (done ? 'Mark as unread' : 'Mark as read') }, + done ? (readAt ? '✓ ' + readAt.text : '✓ read') : p ? `page ${p.p + 1} / ${p.n}` : 'unread'); state.onclick = async () => { - if (done) { await api('api/unread', { m: M.id, c: ch.id }); ch.progress = null; } + if (done) { await api('api/unread', { m: M.id, c: ch.id }); ch.progress = null; ch.readAt = null; } else { const version = ch.versions[0]; await api('api/progress', { m: M.id, c: ch.id, v: version.id, p: version.pages - 1, n: version.pages, done: true }); ch.progress = { v: version.id, p: version.pages - 1, n: version.pages, done: true }; + ch.readAt = { at: Date.now(), approx: false }; } draw(); }; @@ -228,12 +230,14 @@ ? h('button', { class: 'btn small', onclick: () => { panel.hidden = !panel.hidden; } }, ch.versions.length + ' versions') : null; const pages = ch.versions[0] ? ch.versions[0].pages : 0; - const timing = chapterTiming(ch); + const timing = chapterTiming(ch), released = chapterRelease(ch.dates); wrap.textContent = ''; wrap.append(h('div', { class: 'ch' + (done ? ' read' : '') }, - h('a', { class: 'ch-main' + (timing ? ' has-time' : ''), href: readUrl(M.id, ch.id) }, + h('a', { class: 'ch-main' + (timing ? ' has-time' : '') + (M.chapterDates?.release ? ' has-date' : ''), href: readUrl(M.id, ch.id) }, h('span', { class: 'ch-label' }, ch.label), h('span', { class: 'ch-pages' }, `${pages} p.`), + M.chapterDates?.release ? h('span', { class: 'ch-date' + (released ? ' tier-' + released.tier : ''), title: released ? released.title : 'No date known', 'aria-label': released ? released.title : 'No date known' }, + released ? released.text : '–') : null, timing ? h('span', { class: 'ch-time', title: timing.title, 'aria-label': timing.title }, timing.text) : null, h('span', { class: 'ch-title', title: ch.title || '' }, ch.title || 'Untitled'), ch.extra ? h('span', { class: 'tag' }, 'extra') : null), @@ -261,6 +265,42 @@ return wrap; } + // Phones (the chapter-row breakpoint in app.css) get numeric dates to leave room for titles. + const compact = matchMedia('(max-width: 640px)').matches; + + // Read date: the day and the hour (local time) the chapter was first completed. + function readDate(value) { + const mode = M.chapterDates?.read; + if (!value || !value.at || mode === 'off') return null; + const when = new Date(value.at), thisYear = when.getFullYear() === new Date().getFullYear(); + const day = when.toLocaleDateString(undefined, compact ? { day: '2-digit', month: '2-digit', ...(thisYear ? {} : { year: '2-digit' }) } + : { day: 'numeric', month: 'short', ...(thisYear ? {} : { year: 'numeric' }) }); + const text = (value.approx ? '≤ ' : '') + day + (mode === 'hour' ? `, ${String(when.getHours()).padStart(2, '0')}h` : ''); + const full = when.toLocaleDateString(undefined, { dateStyle: 'long' }) + (mode === 'hour' ? `, ${String(when.getHours()).padStart(2, '0')}:00–${String((when.getHours() + 1) % 24).padStart(2, '0')}:00` : ''); + return { text, title: (value.approx ? 'Read by ' : 'Read on ') + full + (value.approx ? ' (earlier reads were not recorded)' : '') }; + } + + // Release date by tier: official > source upload > Manga Circus ecosystem > this library. + const TIER = { official: { mark: '' }, source: { mark: '≈' }, ecosystem: { mark: '◎' }, library: { mark: '↓' } }; + const dayText = (at, precision, long) => { + // Day-precision dates are calendar dates, not instants: show them in UTC so they do not shift. + const dayOnly = !String(at).includes('T'); + const opts = precision === 'year' ? { year: 'numeric' } : precision === 'month' ? { year: 'numeric', month: 'short' } + : long ? { dateStyle: 'long' } : compact ? { day: '2-digit', month: '2-digit', year: '2-digit' } : { day: 'numeric', month: 'short', year: 'numeric' }; + return new Date(at).toLocaleDateString(undefined, { ...opts, ...(dayOnly ? { timeZone: 'UTC' } : {}) }); + }; + function chapterRelease(dates) { + if (!dates || !dates.release) return null; + const known = dates.known || {}, lines = []; + if (known.official) lines.push(`Released ${dayText(known.official.at, known.official.precision, true)}${known.official.name ? ' (' + known.official.name + ')' : ''}`); + if (known.source) lines.push(`Uploaded at ${known.source.name || 'the source'} ${dayText(known.source.at, 'day', true)}`); + if (known.batch) lines.push(`Bulk upload at ${known.batch.name || 'the source'} ${dayText(known.batch.at, 'day', true)} (not a release date)`); + if (known.ecosystem) lines.push(`First in Manga Circus ${dayText(known.ecosystem.at, 'day', true)}${known.ecosystem.name && known.ecosystem.name !== 'this server' ? ' (via ' + known.ecosystem.name + ')' : ''}`); + if (known.library) lines.push(`Added to this library ${dayText(known.library.at, 'day', true)}`); + const tier = TIER[dates.tier] || TIER.library; + return { tier: dates.tier, text: tier.mark + dayText(dates.release.at, dates.release.precision), title: lines.join('\n') }; + } + function chapterTiming(chapter) { if (!M.reading) return null; const value = chapter.reading; diff --git a/public/settings.js b/public/settings.js index 679c1e4..ae8e7e9 100644 --- a/public/settings.js +++ b/public/settings.js @@ -38,6 +38,11 @@ h('option', { value: 'dim' }, 'Dim (70% brightness)'), h('option', { value: 'extra-dim' }, 'Extra dim (45% brightness)')); tone.value = S.pageTone; + const readDate = h('select', { onchange: e => save({ readDate: e.target.value }) }, + h('option', { value: 'hour' }, 'Date and hour (e.g. 23 Oct, 21h)'), + h('option', { value: 'date' }, 'Date only'), + h('option', { value: 'off' }, 'Do not show')); + readDate.value = S.readDate; const LABEL = { flow: 'Flow', page: 'Page', book: 'Book', strip: 'Webtoon flow' }; const zooms = () => Object.keys(LABEL).map(k => `${LABEL[k]} ${Math.round((S.zoom[k] || 1) * 100)}%`).join(' · '); const zoomText = h('span', { class: 'muted' }, zooms()); @@ -85,6 +90,13 @@ h('button', { class: 'btn small', onclick: async () => { await save({ zoom: {} }); zoomText.textContent = zooms(); } }, 'Reset to fit'), h('div', { class: 'muted small' }, '100% = the page fits the screen. Change it in the reader with Ctrl + mouse wheel, a pinch, or the + / - keys.')))); + section('user/reading').append(h('section', { class: 'card' }, + h('h2', {}, 'Chapter list dates'), + h('label', { class: 'field' }, h('span', {}, 'When I read a chapter'), readDate, + h('div', { class: 'muted small' }, 'Shown next to ✓ in the chapter list: when you first finished the chapter, in your local time. Marking it unread clears it.')), + check('releaseDate', 'Show when each chapter was released', + 'Best known date first: the official release, else the upload at the source (≈), else when it first reached Manga Circus (◎), else when it was added here (↓). Hover a date for all known dates.'))); + const readingHoursBody = h('div', {}, h('p', { class: 'muted' }, 'Loading server reading hours…')); section('user/reading-hours').append(h('section', { class: 'card' }, h('h2', {}, 'Server reading hours'), readingHoursBody, diff --git a/reading-analytics.js b/reading-analytics.js index ac30d49..9b5cf33 100644 --- a/reading-analytics.js +++ b/reading-analytics.js @@ -115,6 +115,17 @@ function chapterInsight(record) { }; } +// Earliest completion time the clock still remembers (only the last few visits are kept). +// approx: older completed passes were already dropped, so the first read may be earlier. +function earliestCompletion(record) { + if (!record || typeof record !== 'object') return null; + const passes = [record.currentPass, ...(record.recentSessions || []).map(item => item && item.pass)]; + const times = passes.map(pass => pass && whole(pass.completedAt)).filter(time => time > 0); + if (!times.length) return null; + const counted = new Set(times).size; + return { at: Math.min(...times), approx: whole(record.completedPasses) > counted }; +} + function median(values) { const sorted = values.filter(value => Number.isFinite(value) && value > 0).sort((a, b) => a - b); if (!sorted.length) return null; @@ -295,6 +306,6 @@ function applyPulse(record, pulse, { version, pageCount, eligible, now = Date.no } module.exports = { - INPUT_TYPES, applyPulse, applyVisit, chapterInsight, compactRanges, eligiblePages, emptyRecord, + INPUT_TYPES, applyPulse, applyVisit, chapterInsight, compactRanges, earliestCompletion, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary, rangeSet, }; diff --git a/scraper/adapters/mangadex.js b/scraper/adapters/mangadex.js index 147e1cd..12f04ad 100644 --- a/scraper/adapters/mangadex.js +++ b/scraper/adapters/mangadex.js @@ -3,6 +3,7 @@ const API = 'https://api.mangadex.org'; const { parse } = require('../plugins/registry'); const { localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary, pageUrls } = require('../plugins/mangadex'); +const { sourceDate } = require('../../chapter-dates'); function mangaDexInfo(value) { let url; @@ -71,8 +72,9 @@ async function discoverMangaDexChapter(ref, http) { const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`); const groupName = ref.group?.name ? ` · ${ref.group.name}` : ''; const parsed = parse('mangadex', 'atHome', { data: { response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }, url: `${API}/at-home/server/${ref.id}` }); + const source = sourceDate(ref.uploadedAt, `MangaDex${groupName}`, ref.uploadBatch); return { - number: ref.number, title: ref.title || '', added: ref.added, + number: ref.number, title: ref.title || '', ...(source ? { dates: { source } } : {}), ...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}), source: { name: `MangaDex${groupName}`, url: ref.url }, pages: parsed.pages, diff --git a/scraper/adapters/weebcentral.js b/scraper/adapters/weebcentral.js index d0e9bbc..06506cc 100644 --- a/scraper/adapters/weebcentral.js +++ b/scraper/adapters/weebcentral.js @@ -4,6 +4,7 @@ const { SlowHttp } = require('../lib/http'); const { fallbackDirection } = require('../../reading-direction'); const parser = require('../plugins/weebcentral'); const { parse } = require('../plugins/registry'); +const { sourceDate } = require('../../chapter-dates'); function weebCentralInfo(value) { let url; @@ -36,7 +37,8 @@ async function discoverWeebCatalog(seriesUrl, http) { async function discoverWeebChapter(chapter, http) { const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text(); - const { manifestUrl, added } = parse('weebcentral', 'chapter', { html: pageHtml, url: chapter.url }); + const { manifestUrl } = parse('weebcentral', 'chapter', { html: pageHtml, url: chapter.url }); + const source = sourceDate(chapter.uploadedAt, 'WeebCentral', chapter.uploadBatch); const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip¤t_page=1`, { accept: 'text/html' })).text(); const parsed = parse('weebcentral', 'pages', { html: fragment, url: manifestUrl }); const pages = parsed.pages.map(url => ({ url, headers: { referer: chapter.url } })); @@ -46,7 +48,7 @@ async function discoverWeebChapter(chapter, http) { ...(chapter.special ? { special: true, label: chapter.label, sourceId: chapter.id, sourceNumber: chapter.volume, releaseKind: chapter.releaseKind, volume: chapter.volume } : {}), ...(chapter.book ? { book: chapter.book } : {}), - added, + ...(source ? { dates: { source } } : {}), source: { name: 'WeebCentral', url: chapter.url }, pages, }; } diff --git a/scraper/lib/publisher.js b/scraper/lib/publisher.js index c1599a2..084e1a1 100644 --- a/scraper/lib/publisher.js +++ b/scraper/lib/publisher.js @@ -12,6 +12,7 @@ const { directionFields } = require('../../reading-direction'); const { mergeImportedMetadata } = require('./metadata-merge'); const { normalizeWorkIdentity } = require('./work-identity'); const { parseKey } = require('../../chapter-identity'); +const { publishedDates } = require('../../chapter-dates'); const token = () => crypto.randomBytes(5).toString('hex'); // Staging is a recovery asset, not disposable garbage. Legacy stages without an @@ -180,7 +181,8 @@ async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) { await fsp.writeFile(path.join(stage, 'chapter.json'), JSON.stringify({ ...priorMeta, title: chapter.title || '', source: chapter.source, - added: chapter.added || new Date().toISOString(), + // "added" stays the first arrival in this library; x-dates keeps source/release/ecosystem. + ...publishedDates(priorMeta, { ...(chapter.dates || {}), added: chapter.added }), priority: Number(chapter.priority || 0), extra: !!chapter.extra, ...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId, sourceNumber: chapter.sourceNumber ?? null, diff --git a/scraper/plugins/mangadex.js b/scraper/plugins/mangadex.js index 0ee2209..4eca919 100644 --- a/scraper/plugins/mangadex.js +++ b/scraper/plugins/mangadex.js @@ -1,6 +1,7 @@ 'use strict'; const { specialKey } = require('../../chapter-identity'); +const { batchFlags } = require('../../chapter-dates'); function localizedTitle(attributes = {}) { const titles = attributes.title || {}; @@ -41,14 +42,18 @@ function selectChapterReleases(resources = [], specialLabel = 'Special') { const prior = selected.get(number); if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource); } - return [...selected.entries()].map(([number, resource]) => { + const chosen = [...selected.entries()]; + const uploaded = chosen.map(([, resource]) => resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt); + const batches = batchFlags(uploaded); + return chosen.map(([number, resource], index) => { const group = groupFor(resource); return { id: resource.id, number, title: resource.attributes?.title || '', ...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}), ...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}), pages: Number(resource.attributes?.pages || 0), - added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt, + // A group's upload time, not an official release; bulk uploads are flagged. + ...(uploaded[index] ? { uploadedAt: uploaded[index], uploadBatch: !!batches[index] } : {}), group, url: `https://mangadex.org/chapter/${resource.id}`, }; diff --git a/scraper/plugins/weebcentral.js b/scraper/plugins/weebcentral.js index 4d01304..dff7815 100644 --- a/scraper/plugins/weebcentral.js +++ b/scraper/plugins/weebcentral.js @@ -1,5 +1,6 @@ 'use strict'; const { volumeKey } = require('../../chapter-identity'); +const { batchFlags } = require('../../chapter-dates'); // First-party, pure parsing functions. This module never fetches remote content. const decode = value => value.replace(/&/g, '&').replace(/'/g, "'").replace(/"/g, '"').replace(/<[^>]+>/g, '').trim(); @@ -21,19 +22,26 @@ function catalog({ html, url }) { const volume = !regular ? text.match(/^Volume\s+([0-9]+(?:\.[0-9]+)?)(?!\d)/i)?.[1] : null; const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1]; const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null); + // The full chapter list carries each upload time; the chapter page itself has none. + const uploaded = match[3].match(/datetime="([^"]+)"/i)?.[1]; + const uploadedAt = uploaded && Number.isFinite(Date.parse(uploaded)) ? uploaded : undefined; if (number) found.push({ id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(), ...(season ? { book: { number: season, title: `Season ${season}` } } : {}), - url: absolute(url, match[1]), raw: text, + url: absolute(url, match[1]), raw: text, ...(uploadedAt ? { uploadedAt } : {}), }); else if (volume) found.push({ id: match[2], number: volumeKey(match[2]), title: '', special: true, label: `Volume ${volume}`, releaseKind: 'volume', volume, book: { number: Number(volume), title: `Volume ${volume}` }, - url: absolute(url, match[1]), raw: text, + url: absolute(url, match[1]), raw: text, ...(uploadedAt ? { uploadedAt } : {}), }); } const chapters = [...new Map(found.map(chapter => [chapter.id, chapter])).values()]; + // Back-catalog chapters were imported together; their upload time is not a release date. + batchFlags(chapters.map(chapter => chapter.uploadedAt)).forEach((batch, index) => { + if (batch !== null) chapters[index].uploadBatch = batch; + }); const chapterLinks = [...html.matchAll(/]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length; return { chapters, chapterLinks }; } @@ -42,7 +50,6 @@ function chapter({ html, url }) { return { manifestUrl: html.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1] || `${url}/images?is_prev=False`, - added: html.match(/datetime="([^"]+)"/i)?.[1], }; } diff --git a/scraper/test/source-plugin.test.js b/scraper/test/source-plugin.test.js index d83c9f1..3628148 100644 --- a/scraper/test/source-plugin.test.js +++ b/scraper/test/source-plugin.test.js @@ -10,8 +10,8 @@ const { adapters, adapterFor } = require('../adapters'); const seriesUrl = 'https://weebcentral.com/series/ABC/title'; const chapterUrl = 'https://weebcentral.com/chapters/XYZ'; const seriesHtml = '

Test Manga

'; -const listHtml = 'Chapter 1Log. 2'; -const chapterHtml = ''; +const listHtml = 'Chapter 1Log. 2'; +const chapterHtml = '
no dates on the chapter page
'; const imagesHtml = ''; test('first-party registry rejects unknown plugins and malformed input', () => { @@ -45,7 +45,9 @@ test('in-process parser preserves WeebCentral catalog and page discovery', async assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']); const chapter = await discoverWeebChapter(catalog.chapters[0], http); assert.equal(chapter.pages.length, 2); - assert.equal(chapter.added, '2026-01-01'); + assert.equal(chapter.added, undefined); + assert.deepEqual(chapter.dates, { source: { at: '2026-01-01T15:06:05.348Z', name: 'WeebCentral', batch: false } }); + assert.equal(catalog.chapters[1].uploadedAt, undefined); assert.equal(requested.length, 4); }); diff --git a/server.js b/server.js index dd485e5..dad2ca3 100644 --- a/server.js +++ b/server.js @@ -25,8 +25,9 @@ const peerClient = require('./peering/client'); const { DiscoveryStore } = require('./discovery/store'); const { generate: generateDiscovery } = require('./discovery/engine'); const { - applyPulse, applyVisit, chapterInsight, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary, + applyPulse, applyVisit, chapterInsight, earliestCompletion, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary, } = require('./reading-analytics'); +const { chapterDates } = require('./chapter-dates'); const { parseSchedule, publicSchedule, loadQuestionBank, createReadingAccess } = require('./reading-access'); const { BackupService, BackupError } = require('./backup'); @@ -97,7 +98,7 @@ function checkPw(pw, stored) { } const DUMMY_HASH = hashPw('x'); const DEFAULTS = { startMode: 'last', lastMode: 'page', adapt: true, skipExtras: false, skipCovers: false, flowPageGap: true, pageTone: 'original', readingClock: false, readingStory: true, - discoveryEnabled: false, discoveryReadingActivity: false, discoveryMature: false, zoom: {}, webhook: '' }; + readDate: 'hour', releaseDate: true, discoveryEnabled: false, discoveryReadingActivity: false, discoveryMature: false, zoom: {}, webhook: '' }; const settingsOf = u => ({ ...DEFAULTS, ...u.settings, zoom: { ...(u.settings && u.settings.zoom) } }); function createUser(name, pw, admin, replace) { @@ -322,6 +323,13 @@ function firstReleaseOf(meta) { } return { label: label || '', time: Number.isFinite(time) ? time : 0 }; } +// When a done chapter was completed: the progress record, or for reads saved before doneAt +// existed, the earliest completion the reading clock still remembers. +function readAtOf(progress, record) { + if (!progress || !progress.done) return null; + if (progress.doneAt) return { at: progress.doneAt, approx: false }; + return earliestCompletion(record); +} function continueOf(m, prog, s) { const ok = c => !(s.skipExtras && c.extra); let last = null; @@ -437,7 +445,8 @@ const routes = { if (['last', 'flow', 'page', 'book'].includes(b.startMode)) s.startMode = b.startMode; if (['flow', 'page', 'book'].includes(b.lastMode)) s.lastMode = b.lastMode; if (['original', 'inverted', 'sepia', 'dim', 'extra-dim'].includes(b.pageTone)) s.pageTone = b.pageTone; - for (const k of ['adapt', 'skipExtras', 'skipCovers', 'flowPageGap', 'readingClock', 'readingStory', + if (['off', 'date', 'hour'].includes(b.readDate)) s.readDate = b.readDate; + for (const k of ['adapt', 'skipExtras', 'skipCovers', 'flowPageGap', 'readingClock', 'readingStory', 'releaseDate', 'discoveryEnabled', 'discoveryReadingActivity', 'discoveryMature']) if (typeof b[k] === 'boolean') s[k] = b[k]; if (b.zoom && typeof b.zoom === 'object') { s.zoom = {}; @@ -557,9 +566,10 @@ const routes = { }, 'GET /api/manga': ({ u, q }) => { - const m = mangaOf(q.get('m')), prog = progressOf(u, m), showStory = settingsOf(u).readingStory; + const m = mangaOf(q.get('m')), prog = progressOf(u, m), s = settingsOf(u), showStory = s.readingStory; const records = mangaReadingOf(u, m); return { + chapterDates: { read: s.readDate, release: s.releaseDate }, id: m.id, title: m.title, meta: m.meta, cover: url(m.cover), direction: m.direction, collections: catalog.state.collections.filter(c => c.members.some(member => member.mangaId === m.id)).map(c => ({ id: c.id, title: c.title })), relations: catalog.state.relations.filter(r => r.state === 'confirmed' && (r.from === m.id || r.to === m.id)).map(r => ({ ...r, @@ -569,6 +579,8 @@ const routes = { chapters: m.chapters.map(ch => ({ id: ch.id, book: ch.book, label: ch.label, title: ch.title, extra: ch.extra, progress: prog[ch.id] || null, versions: versionsOf(m, ch, u), reading: showStory ? chapterInsight(records[ch.id]) : null, + readAt: s.readDate === 'off' ? null : readAtOf(prog[ch.id], records[ch.id]), + dates: s.releaseDate ? chapterDates(ch.versions) : null, })), reading: showStory ? mangaInsight(records, m.chapters.map(ch => ch.id)) : null, continue: continueOf(m, prog, settingsOf(u)), @@ -637,8 +649,11 @@ const routes = { 'POST /api/progress': ({ u, b }) => { const m = mangaOf(b.m), ch = chapterOf(m, b.c); const all = (db.progress[u.name] = db.progress[u.name] || {}), mp = (all[m.id] = all[m.id] || {}), old = mp[ch.id] || {}; + const now = Date.now(), done = !!(old.done || b.done); + // doneAt: when the chapter was first completed; kept until it is marked unread. + const doneAt = old.done ? old.doneAt : done ? now : undefined; mp[ch.id] = { - v: String(b.v || ''), p: Math.max(0, b.p | 0), n: Math.max(0, b.n | 0), t: Date.now(), done: !!(old.done || b.done), + v: String(b.v || ''), p: Math.max(0, b.p | 0), n: Math.max(0, b.n | 0), t: now, done, ...(doneAt ? { doneAt } : {}), shift: Array.isArray(b.shift) ? b.shift.filter(Number.isInteger).slice(0, 1000) : old.shift || [], }; save(); diff --git a/test/chapter-dates.test.js b/test/chapter-dates.test.js new file mode 100644 index 0000000..cb239c6 --- /dev/null +++ b/test/chapter-dates.test.js @@ -0,0 +1,81 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { batchFlags, publishedDates, chapterDates, peerDates, fromPeer } = require('../chapter-dates'); +const { earliestCompletion } = require('../reading-analytics'); + +test('bulk imports are flagged, weekly uploads are not', () => { + const flags = batchFlags([ + '2024-09-07T10:00:00Z', '2024-09-07T10:00:03Z', '2024-09-07T10:04:00Z', // one import + '2026-10-11T15:05:58Z', '2026-10-18T15:06:05Z', // weekly releases + '2026-10-20T09:00:00Z', '2026-10-20T09:01:00Z', // a double release stays a release + undefined, + ]); + assert.deepEqual(flags, [true, true, true, false, false, false, false, null]); +}); + +test('publishing keeps the first arrival and the earliest ecosystem date', () => { + const now = Date.parse('2026-10-24T12:00:00Z'); + const fresh = publishedDates({}, { source: { at: '2026-10-18T15:06:05Z', name: 'WeebCentral', batch: false } }, now); + assert.equal(fresh.added, '2026-10-24T12:00:00.000Z'); + assert.deepEqual(fresh['x-dates'].ecosystem, { at: '2026-10-24T12:00:00.000Z', via: 'this server' }); + assert.equal(fresh['x-dates'].source.name, 'WeebCentral'); + + // Replacing a version later: "added" and the ecosystem date survive. + const replaced = publishedDates(fresh, {}, now + 864e5); + assert.equal(replaced.added, fresh.added); + assert.deepEqual(replaced['x-dates'].ecosystem, fresh['x-dates'].ecosystem); + assert.equal(replaced['x-dates'].source.at, '2026-10-18T15:06:05.000Z'); + + // A peer that had it earlier lowers the date; a later claim never raises it. + const earlier = publishedDates(fresh, { ecosystem: { at: '2026-10-19T08:00:00Z', via: 'peer bob' } }, now); + assert.deepEqual(earlier['x-dates'].ecosystem, { at: '2026-10-19T08:00:00.000Z', via: 'peer bob' }); + const later = publishedDates(earlier, { ecosystem: { at: '2026-10-23T00:00:00Z', via: 'peer eve' } }, now); + assert.equal(later['x-dates'].ecosystem.via, 'peer bob'); +}); + +test('peer ecosystem claims cannot predate the source upload or lie in the future', () => { + const now = Date.parse('2026-10-24T12:00:00Z'); + const source = { at: '2026-10-18T15:06:05Z', name: 'WeebCentral', batch: false }; + const early = publishedDates({}, { source, ecosystem: { at: '2001-01-01T00:00:00Z', via: 'peer x' } }, now); + assert.equal(early['x-dates'].ecosystem.at, '2026-10-18T15:06:05.000Z'); + const future = publishedDates({}, { ecosystem: { at: '2099-01-01T00:00:00Z', via: 'peer x' } }, now); + assert.equal(future['x-dates'].ecosystem.at, '2026-10-24T12:00:00.000Z'); +}); + +test('release date tiers: official, source upload, ecosystem, library', () => { + const added = Date.parse('2026-10-17T15:04:03Z'); + const version = dates => ({ added, meta: { 'x-dates': dates } }); + const ecosystem = { at: '2026-10-10T00:00:00Z', via: 'peer bob' }; + assert.equal(chapterDates([version({})]).tier, 'library'); + assert.equal(chapterDates([version({ ecosystem })]).release.at, '2026-10-10T00:00:00.000Z'); + const batch = chapterDates([version({ ecosystem, source: { at: '2024-09-07T10:00:00Z', name: 'WeebCentral', batch: true } })]); + assert.equal(batch.tier, 'ecosystem', 'a bulk upload is no release date'); + assert.equal(batch.known.batch.at, '2024-09-07T10:00:00.000Z'); + const source = chapterDates([version({ ecosystem }), version({ source: { at: '2026-10-04T15:06:05Z', name: 'MangaDex', batch: false } })]); + assert.equal(source.tier, 'source'); + assert.equal(source.known.ecosystem.at, '2026-10-10T00:00:00.000Z', 'earliest across versions'); + const official = chapterDates([version({ release: { at: '2023-09-17', precision: 'day', source: 'MangaUpdates · Viz' }, ecosystem })]); + assert.deepEqual(official.release, { at: '2023-09-17', precision: 'day', name: 'MangaUpdates · Viz' }); + assert.equal(official.tier, 'official'); +}); + +test('peers exchange cleaned dates and the receiver records the peer', () => { + const meta = { added: '2026-10-17T15:04:03Z', secret: 'x', + 'x-dates': { source: { at: '2026-10-18T15:06:05Z', name: 'WeebCentral', batch: false, extra: 'drop' } } }; + const sent = peerDates(meta, 0); + assert.deepEqual(sent, { ecosystem: '2026-10-17T15:04:03.000Z', source: { at: '2026-10-18T15:06:05.000Z', name: 'WeebCentral', batch: false } }); + const received = fromPeer(sent, 'bob'); + assert.deepEqual(received.ecosystem, { at: '2026-10-17T15:04:03.000Z', via: 'peer bob' }); + assert.deepEqual(fromPeer('nonsense', 'bob'), {}); + assert.deepEqual(fromPeer({ ecosystem: 'not a date' }, 'bob'), {}); +}); + +test('legacy reads get the earliest completion the reading clock remembers', () => { + assert.equal(earliestCompletion(null), null); + const record = { completedPasses: 3, currentPass: { completedAt: 5000 }, + recentSessions: [{ pass: { completedAt: 3000 } }, { pass: { completedAt: null } }, { pass: { completedAt: 5000 } }] }; + assert.deepEqual(earliestCompletion(record), { at: 3000, approx: true }); + assert.deepEqual(earliestCompletion({ ...record, completedPasses: 2 }), { at: 3000, approx: false }); +}); diff --git a/tools/backfill-chapter-dates.js b/tools/backfill-chapter-dates.js new file mode 100644 index 0000000..446aa11 --- /dev/null +++ b/tools/backfill-chapter-dates.js @@ -0,0 +1,83 @@ +#!/usr/bin/env node +'use strict'; +// One-time migration to chapter dates (BL-0047). Dry run by default; apply only with the +// scraper stopped (--apply --writers-stopped), then run a deep "Rescan" in Settings. +// - WeebCentral: x-dates.source from the series' full chapter list (one request per series). +// - MangaDex: chapters published before x-dates stored the group's publishAt in "added"; +// it moves to x-dates.source and "added" becomes the folder's arrival time. +// - Every chapter: x-dates.ecosystem = "added" unless an earlier one is already known. +const fs = require('node:fs').promises; +const path = require('node:path'); +const { batchFlags, sourceDate, publishedDates } = require('../chapter-dates'); +const weebcentral = require('../scraper/plugins/weebcentral'); +const { writeJsonAtomic } = require('../scraper/lib/publisher'); + +const args = process.argv.slice(2); +const apply = args.includes('--apply'); +const only = args.find(arg => arg.startsWith('--manga='))?.slice('--manga='.length); +const offline = args.includes('--offline'); +const root = args.find(arg => !arg.startsWith('--')); +if (!root || args.some(arg => arg.startsWith('--') && !['--apply', '--writers-stopped', '--offline'].includes(arg) && !arg.startsWith('--manga=')) + || (apply && !args.includes('--writers-stopped'))) { + console.error('Usage: node tools/backfill-chapter-dates.js [--manga=] [--offline] [--apply --writers-stopped]'); + process.exit(2); +} +const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130 Safari/537.36'; +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const readJson = async file => { try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; } }; +const dirs = async dir => (await fs.readdir(dir, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name).sort(); + +async function weebCentralUploads(seriesId) { + const url = `https://weebcentral.com/series/${seriesId}/full-chapter-list`; + const response = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, signal: AbortSignal.timeout(30000) }); + if (!response.ok) throw new Error(`WeebCentral HTTP ${response.status}`); + const { chapters } = weebcentral.catalog({ html: await response.text(), url }); + return new Map(chapters.filter(c => c.uploadedAt).map(c => [c.id, sourceDate(c.uploadedAt, 'WeebCentral', c.uploadBatch)])); +} + +async function main() { + const totals = { chapters: 0, changed: 0, alreadyDated: 0, weebSource: 0, weebBatch: 0, mangaDexMoved: 0, missing: 0 }; + for (const manga of await dirs(root)) { + if (only && manga !== only) continue; + const mangaDir = path.join(root, manga), meta = (await readJson(path.join(mangaDir, 'manga.json'))) || {}; + const versions = []; + for (const book of await dirs(mangaDir)) { + for (const chapter of await dirs(path.join(mangaDir, book))) { + const dir = path.join(mangaDir, book, chapter), file = path.join(dir, 'chapter.json'), chapterMeta = await readJson(file); + if (chapterMeta) versions.push({ dir, file, meta: chapterMeta, mtime: (await fs.stat(dir)).mtimeMs }); + } + } + let uploads = new Map(); + const seriesId = meta['x-scraper']?.ids?.weebcentral; + if (seriesId && !offline && versions.some(v => /weebcentral\.com\/chapters\//.test(v.meta.source?.url || '') && !v.meta['x-dates'])) { + try { uploads = await weebCentralUploads(seriesId); await sleep(2000); } + catch (error) { console.warn(`[backfill] ${manga}: WeebCentral list failed: ${error.message}`); } + } + // Legacy MangaDex "added" values are group upload times; flag bulk uploads among them. + const legacyDex = versions.filter(v => !v.meta['x-dates'] && /^MangaDex\b/.test(v.meta.source?.name || '') && Date.parse(v.meta.added)); + const dexBatch = new Map(batchFlags(legacyDex.map(v => v.meta.added)).map((flag, i) => [legacyDex[i], flag])); + let changed = 0; + for (const version of versions) { + totals.chapters++; + if (version.meta['x-dates']) { totals.alreadyDated++; continue; } + const prior = { ...version.meta }, incoming = {}; + const weebId = /weebcentral\.com\/chapters\/([0-9A-Z]{26})/.exec(prior.source?.url || '')?.[1]; + if (weebId && uploads.has(weebId)) { + incoming.source = uploads.get(weebId); + totals[incoming.source.batch ? 'weebBatch' : 'weebSource']++; + } else if (dexBatch.has(version)) { + incoming.source = sourceDate(prior.added, prior.source.name, dexBatch.get(version)); + prior.added = new Date(version.mtime).toISOString(); + totals.mangaDexMoved++; + } else totals.missing++; + const next = { ...prior, ...publishedDates(prior, incoming, version.mtime) }; + changed++; totals.changed++; + if (apply) await writeJsonAtomic(version.file, next); + } + if (changed) console.log(`[backfill] ${manga}: ${changed} of ${versions.length} chapter versions ${apply ? 'updated' : 'would change'}`); + } + console.log(JSON.stringify({ apply, ...totals })); + if (apply) console.log('Run a deep "Rescan" in Settings so the reader picks up the edited chapter.json files.'); +} + +main().catch(error => { console.error(error); process.exit(1); });