Chapter dates: read date setting and tiered release dates (BL-0047)
- Chapter rows show when you first finished a chapter (setting: date + hour, date, off) from a new durable progress doneAt; older reads fall back to the reading clock's earliest remembered completion. - chapter.json keeps added (arrival in this library, now kept on replace) plus x-dates.source (source upload, bulk imports flagged), x-dates.release (official, for a later metadata job) and x-dates.ecosystem (first entry into any Manga Circus library; carried over peering, earliest wins, clamped). - Rows show one release date by tier: official > source upload > ecosystem > library, with all known dates in the tooltip (setting to hide). - Fixes: WeebCentral dates came from the chapter page, which has none (every chapter got the download time); they now come from the full chapter list. MangaDex no longer writes a group's publishAt into added. Peering sends dates. - tools/backfill-chapter-dates.js migrates an existing library (dry run default). - Coverage dry run on the server: docs/chapter-dates-coverage-2026-10-24.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
MaashkinoozheandClaude Opus 5.5 committed
1 parent
b2bb34174c
commit
e0dacd81c6
21 files changed
+470
-32
No files matched your search
@@ -5,6 +5,7 @@
|
||||
!library.js
|
||||
!reading-direction.js
|
||||
!chapter-identity.js
|
||||
!chapter-dates.js
|
||||
!catalog.js
|
||||
!backup.js
|
||||
!reading-analytics.js
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ RUN apk add --no-cache tzdata openssl
|
||||
WORKDIR /app
|
||||
ARG VCS_REF=development
|
||||
LABEL org.opencontainers.image.title="MangaCircus" org.opencontainers.image.revision=$VCS_REF
|
||||
COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js reading-access.js catalog.js backup.js ./
|
||||
COPY package.json server.js library.js reading-direction.js chapter-identity.js chapter-dates.js reading-analytics.js reading-access.js catalog.js backup.js ./
|
||||
COPY public ./public
|
||||
COPY greatest ./greatest
|
||||
COPY discovery ./discovery
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
'use strict';
|
||||
// Chapter dates (BL-0047). chapter.json keeps three independent dates:
|
||||
// added when the chapter entered this library (kept on replace)
|
||||
// x-dates.source when the source site uploaded it; batch=true marks a bulk import, not a release
|
||||
// x-dates.release the official public release (filled by a metadata job; precision "day")
|
||||
// x-dates.ecosystem when it first entered any Manga Circus library; travels over peering, earliest wins
|
||||
// The reader shows one release date per chapter by tier: official > source (not batch) > ecosystem > library.
|
||||
|
||||
const BATCH_SIZE = 3; // this many uploads ...
|
||||
const BATCH_GAP_MS = 10 * 60e3; // ... each within 10 minutes of the next form a bulk import
|
||||
|
||||
const iso = value => {
|
||||
const time = typeof value === 'number' ? value : Date.parse(value);
|
||||
return Number.isFinite(time) && time > 0 ? new Date(time).toISOString() : null;
|
||||
};
|
||||
const text = (value, max = 120) => (typeof value === 'string' ? value.trim().slice(0, max) : '');
|
||||
|
||||
// Returns one flag per item: true when its upload time belongs to a bulk import, null without a time.
|
||||
function batchFlags(times) {
|
||||
const order = times.map((value, index) => ({ index, time: Date.parse(value) }))
|
||||
.filter(item => Number.isFinite(item.time)).sort((a, b) => a.time - b.time);
|
||||
const flags = times.map(() => null);
|
||||
for (let start = 0; start < order.length;) {
|
||||
let end = start + 1;
|
||||
while (end < order.length && order[end].time - order[end - 1].time <= BATCH_GAP_MS) end++;
|
||||
for (let i = start; i < end; i++) flags[order[i].index] = end - start >= BATCH_SIZE;
|
||||
start = end;
|
||||
}
|
||||
return flags;
|
||||
}
|
||||
|
||||
function sourceDate(at, name, batch) {
|
||||
const value = iso(at);
|
||||
return value ? { at: value, name: text(name), batch: !!batch } : null;
|
||||
}
|
||||
|
||||
function cleanSource(value) {
|
||||
return value && typeof value === 'object' ? sourceDate(value.at, value.name, value.batch) : null;
|
||||
}
|
||||
function cleanRelease(value) {
|
||||
if (!value || typeof value !== 'object' || !iso(value.at)) return null;
|
||||
const precision = ['day', 'month', 'year'].includes(value.precision) ? value.precision : 'day';
|
||||
return { at: String(value.at).slice(0, 30), precision, source: text(value.source) };
|
||||
}
|
||||
function cleanEcosystem(value) {
|
||||
if (!value || typeof value !== 'object' || !iso(value.at)) return null;
|
||||
return { at: iso(value.at), via: text(value.via) || 'this server' };
|
||||
}
|
||||
|
||||
// chapter.json fields for a (re)published chapter. A prior "added" and an earlier ecosystem
|
||||
// date always survive; a peer's ecosystem claim may lower the date but never raise it.
|
||||
function publishedDates(priorMeta = {}, incoming = {}, now = Date.now()) {
|
||||
const prior = priorMeta['x-dates'] && typeof priorMeta['x-dates'] === 'object' ? priorMeta['x-dates'] : {};
|
||||
const added = iso(priorMeta.added) || iso(incoming.added) || iso(now);
|
||||
const source = cleanSource(incoming.source) || cleanSource(prior.source);
|
||||
const release = cleanRelease(incoming.release) || cleanRelease(prior.release);
|
||||
let claim = cleanEcosystem(incoming.ecosystem);
|
||||
if (claim) {
|
||||
// Nobody can have had the chapter before the source uploaded it, or in the future.
|
||||
const floor = source ? Date.parse(source.at) : 0, time = Math.min(Date.parse(claim.at), now);
|
||||
claim = { ...claim, at: iso(Math.max(time, floor)) };
|
||||
}
|
||||
const candidates = [cleanEcosystem(prior.ecosystem), claim, { at: added, via: 'this server' }].filter(Boolean);
|
||||
const ecosystem = candidates.reduce((best, item) => (Date.parse(item.at) < Date.parse(best.at) ? item : best));
|
||||
return { added, 'x-dates': { version: 1, ...(source ? { source } : {}), ...(release ? { release } : {}), ecosystem } };
|
||||
}
|
||||
|
||||
// Dates of one library chapter from its versions ({ added: ms, meta: chapter.json }).
|
||||
function chapterDates(versions) {
|
||||
const known = {};
|
||||
const keep = (slot, time, value) => {
|
||||
if (Number.isFinite(time) && (!known[slot] || time < known[slot].time)) known[slot] = { time, ...value };
|
||||
};
|
||||
for (const version of versions) {
|
||||
const dates = (version.meta && version.meta['x-dates']) || {};
|
||||
const release = cleanRelease(dates.release), source = cleanSource(dates.source), ecosystem = cleanEcosystem(dates.ecosystem);
|
||||
if (release) keep('official', Date.parse(release.at), { at: release.at, precision: release.precision, name: release.source });
|
||||
if (source) keep(source.batch ? 'batch' : 'source', Date.parse(source.at), { at: source.at, name: source.name });
|
||||
if (ecosystem) keep('ecosystem', Date.parse(ecosystem.at), { at: ecosystem.at, name: ecosystem.via });
|
||||
if (Number.isFinite(version.added)) keep('library', version.added, { at: iso(version.added) });
|
||||
}
|
||||
const tier = ['official', 'source', 'ecosystem', 'library'].find(slot => known[slot]) || null;
|
||||
const out = {};
|
||||
for (const [slot, { time, ...value }] of Object.entries(known)) out[slot] = value;
|
||||
return { tier, release: tier ? out[tier] : null, known: out };
|
||||
}
|
||||
|
||||
// Ecosystem dates offered to peers: the earliest per chapter number, falling back to "added"
|
||||
// for chapters published before x-dates existed (this server was then the one that had it).
|
||||
function ecosystemOf(meta, fallbackMs) {
|
||||
const dates = (meta && meta['x-dates']) || {};
|
||||
return cleanEcosystem(dates.ecosystem)?.at || iso(meta && meta.added) || iso(fallbackMs);
|
||||
}
|
||||
// What a gateway sends with a chapter: only cleaned, non-identifying date fields.
|
||||
function peerDates(meta, fallbackMs) {
|
||||
const dates = (meta && meta['x-dates']) || {};
|
||||
const source = cleanSource(dates.source), release = cleanRelease(dates.release), ecosystem = ecosystemOf(meta, fallbackMs);
|
||||
return { ...(ecosystem ? { ecosystem } : {}), ...(source ? { source } : {}), ...(release ? { release } : {}) };
|
||||
}
|
||||
// Incoming peer dates in publishedDates() shape.
|
||||
function fromPeer(value, alias) {
|
||||
if (!value || typeof value !== 'object') return {};
|
||||
const ecosystem = iso(value.ecosystem);
|
||||
return { ...(ecosystem ? { ecosystem: { at: ecosystem, via: `peer ${text(alias, 40)}`.trim() } } : {}),
|
||||
...(cleanSource(value.source) ? { source: cleanSource(value.source) } : {}),
|
||||
...(cleanRelease(value.release) ? { release: cleanRelease(value.release) } : {}) };
|
||||
}
|
||||
|
||||
module.exports = { BATCH_SIZE, BATCH_GAP_MS, batchFlags, sourceDate, publishedDates, chapterDates, ecosystemOf, peerDates, fromPeer };
|
||||
+29
-1
@@ -208,11 +208,39 @@ An AniList refresh with a different ID is stopped for review before chapter publ
|
||||
|-------|------|---------|
|
||||
| `title` | string | Chapter title without "Chapter N" (default: title part of the folder name) |
|
||||
| `source` | `{name, url}` or string | Where this version came from (default: `[source]` in the folder name) |
|
||||
| `added` | ISO date/time string | Release (or download) date, used for "recently updated" (default: the folder's modification time) |
|
||||
| `added` | ISO date/time string | When this version entered **this** library, used for "recently updated" (default: the folder's modification time). The scraper keeps it when it replaces a version |
|
||||
| `x-dates` | object | Other chapter dates, written by the scraper (see below) |
|
||||
| `extra` | boolean | Marks the chapter as extra without moving it |
|
||||
| `priority` | number | Tie-breaker between versions with equal votes. Higher wins (default `0`) |
|
||||
| `direction` | as above | Overrides book/manga direction, e.g. a colour webtoon special inside a printed series |
|
||||
|
||||
#### Chapter dates (`x-dates`)
|
||||
|
||||
```json
|
||||
"added": "2026-10-17T15:04:03.424Z",
|
||||
"x-dates": {
|
||||
"version": 1,
|
||||
"source": { "at": "2026-10-18T15:06:05.348Z", "name": "WeebCentral", "batch": false },
|
||||
"release": { "at": "2023-09-17", "precision": "day", "source": "MangaUpdates · Viz" },
|
||||
"ecosystem": { "at": "2026-10-17T15:04:03.424Z", "via": "this server" }
|
||||
}
|
||||
```
|
||||
|
||||
- `source`: when the source site uploaded this version (WeebCentral's full chapter list,
|
||||
MangaDex `publishAt`). `batch: true` means it arrived with 3 or more other uploads
|
||||
less than 10 minutes apart, which is a site import, not a release.
|
||||
- `release`: the official public release (`precision` `day`, `month` or `year`). Not
|
||||
filled yet; a metadata job will add it.
|
||||
- `ecosystem`: when the chapter first entered any Manga Circus library. Peer transfers
|
||||
carry it, and the earliest known date always wins. A peer's claim can't be earlier than
|
||||
the source upload or later than now.
|
||||
|
||||
The chapter list shows one release date per chapter, choosing the first tier that has
|
||||
a date: `release` → `source` unless `batch` (shown as ≈) → `ecosystem` (◎) → `added` (↓).
|
||||
Each tier takes the earliest value across the chapter's versions. Existing libraries get
|
||||
these fields from `node tools/backfill-chapter-dates.js <library>`: a dry run by default,
|
||||
then `--apply --writers-stopped` and a deep Rescan.
|
||||
|
||||
Manual directions have priority at **chapter → book → manga**. A `directionOverride`
|
||||
or a plain `direction` without matching generated provenance is manual. Otherwise,
|
||||
the scanner checks the version's existing image dimensions for coherent tall strips
|
||||
|
||||
+9
-2
@@ -110,8 +110,15 @@ an admin can distinguish pairing from permission.
|
||||
| `GET /peer/v1/access` | Pairing state, certificate expiry and current grant status/scopes; no catalog |
|
||||
| `GET /peer/v1/catalog?cursor=N` | Up to 50 approved opaque title IDs, safe metadata, a stable work identity and cover availability |
|
||||
| `GET /peer/v1/covers/{mangaId}` | Approved cover image only; bounded to 5 MiB and charged to peer byte quota |
|
||||
| `GET /peer/v1/manga/{id}?cursor=N` | Up to 100 complete chapter IDs, public metadata and the same stable work identity |
|
||||
| `GET /peer/v1/manga/{id}/chapters/{id}` | Ordered page IDs, MIME, byte size, SHA-256 and ETag |
|
||||
| `GET /peer/v1/manga/{id}?cursor=N` | Up to 100 complete chapter IDs, public metadata, chapter `dates` and the same stable work identity |
|
||||
| `GET /peer/v1/manga/{id}/chapters/{id}` | Ordered page IDs, MIME, byte size, SHA-256 and ETag, plus chapter `dates` |
|
||||
|
||||
Chapter `dates` are `{ ecosystem, source?, release? }`. `ecosystem` is the earliest
|
||||
date at which any version of that chapter number entered a Manga Circus library known
|
||||
to the sender. Older chapters fall back to their `added` date. The receiver stores the
|
||||
date as `x-dates.ecosystem` (`via: "peer <alias>"`) and only lets it lower an existing
|
||||
date, never raise it. See `docs/library-format.md`. Older peers omit `dates`, and the
|
||||
receiver then uses its own arrival time.
|
||||
| `GET /peer/v1/blobs/{id}` | One image, optional `Range: bytes=N-` and `If-Match`; 206 on resume |
|
||||
|
||||
Unknown/unshared IDs return 404. Revoked serving returns 403; bad/missing grants
|
||||
|
||||
@@ -12,6 +12,7 @@ const { publishManifest } = require('../scraper/lib/publisher');
|
||||
const { writeJsonAtomic } = require('../scraper/lib/json-file');
|
||||
const { sleep } = require('../scraper/lib/delay');
|
||||
const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity');
|
||||
const { fromPeer } = require('../chapter-dates');
|
||||
|
||||
const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
|
||||
const store = () => new PeeringState(dataRoot());
|
||||
@@ -132,6 +133,8 @@ async function run(info, item, options) {
|
||||
}
|
||||
const publication = await publishManifest({ version: 1, series, chapters: [{
|
||||
number: ref.number, title: ref.title, book: { number: Number(ref.volume) || 1 },
|
||||
// The peer's ecosystem date travels along: the chapter entered Circus there, not here.
|
||||
dates: fromPeer(chapter.dates || ref.dates, peer.alias),
|
||||
source: { name: `Peer ${peer.alias} ${peer.id.slice(1, 7)}`, url: sourceUrl },
|
||||
pages: pages.map(page => ({ path: page.path })),
|
||||
}] }, { libraryRoot: options.libraryRoot, ...(manga ? { mangaDir: manga.path } : {}) });
|
||||
|
||||
+15
-3
@@ -9,6 +9,7 @@ const crypto = require('node:crypto');
|
||||
const { once } = require('node:events');
|
||||
const { PeeringState, fp, hash, requirePeer, grantStatus } = require('./state');
|
||||
const { parseKey, parseSpecialFolder } = require('../chapter-identity');
|
||||
const { peerDates } = require('../chapter-dates');
|
||||
const { resolveWorkIdentity } = require('../scraper/lib/work-identity');
|
||||
|
||||
const DATA = path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
|
||||
@@ -116,12 +117,23 @@ async function mangaIndex(peer, share) {
|
||||
if (!pages.length) continue;
|
||||
const id = `c${opaque(`${share.id}/${bookName}/${chapterName}`)}`;
|
||||
const chapterMeta = await jsonFile(path.join(chapterDir, 'chapter.json'));
|
||||
const dirStat = await fsp.stat(chapterDir);
|
||||
const item = { id, number: parsed.key, volume: book.key, title: String(chapterMeta.title || parsed.title || '').slice(0, 200),
|
||||
pageCount: pages.length, chapterDir, pages };
|
||||
pageCount: pages.length, chapterDir, pages, dates: peerDates(chapterMeta, dirStat.mtimeMs) };
|
||||
chapters.push(item); byId.set(id, item);
|
||||
if (chapters.length > 10000) fail(413, 'Shared manga chapter limit exceeded');
|
||||
}
|
||||
}
|
||||
// Every version of a chapter number offers the earliest ecosystem date known for it.
|
||||
const earliest = new Map();
|
||||
for (const item of chapters) {
|
||||
const key = `${item.volume}/${item.number}`, at = item.dates.ecosystem;
|
||||
if (at && (!earliest.has(key) || Date.parse(at) < Date.parse(earliest.get(key)))) earliest.set(key, at);
|
||||
}
|
||||
for (const item of chapters) {
|
||||
const at = earliest.get(`${item.volume}/${item.number}`);
|
||||
if (at) item.dates = { ...item.dates, ecosystem: at };
|
||||
}
|
||||
const result = { at: Date.now(), mangaDir, meta, chapters, byId };
|
||||
indexCache.set(key, result);
|
||||
return result;
|
||||
@@ -147,7 +159,7 @@ async function chapterManifest(peer, share, chapterId) {
|
||||
size: stat.size, mtimeMs: stat.mtimeMs, at: Date.now() });
|
||||
if (pages.length > 500) fail(413, 'Chapter page limit exceeded');
|
||||
}
|
||||
return { id: chapter.id, number: chapter.number, volume: chapter.volume, title: chapter.title, pages };
|
||||
return { id: chapter.id, number: chapter.number, volume: chapter.volume, title: chapter.title, dates: chapter.dates, pages };
|
||||
}
|
||||
async function streamBlob(req, res, peer, id) {
|
||||
const item = blobCache.get(`${peer.id}:${id}`);
|
||||
@@ -248,7 +260,7 @@ const server = https.createServer({ key: identity.key, cert: identity.cert, requ
|
||||
return json(res, 200, { id: share.id, title: String(index.meta.title || share.mangaId).slice(0, 200),
|
||||
workIdentity: resolveWorkIdentity(index.meta, share.mangaId, identity.id),
|
||||
...publicMetadata(index.meta), coverAvailable: !!coverFile(index.mangaDir),
|
||||
chapters: index.chapters.slice(cursor, cursor + 100).map(({ id, number, volume, title, pageCount }) => ({ id, number, volume, title, pageCount })),
|
||||
chapters: index.chapters.slice(cursor, cursor + 100).map(({ id, number, volume, title, pageCount, dates }) => ({ id, number, volume, title, pageCount, dates })),
|
||||
nextCursor: cursor + 100 < index.chapters.length ? String(cursor + 100) : null });
|
||||
}
|
||||
const chapterMatch = /^\/peer\/v1\/manga\/(m[0-9a-f]{24})\/chapters\/(c[0-9a-f]{32})$/.exec(url.pathname);
|
||||
|
||||
@@ -68,7 +68,8 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t
|
||||
await fsp.mkdir(chapterDir, { recursive: true });
|
||||
await fsp.writeFile(path.join(mangaDir, 'manga.json'), JSON.stringify({ title: 'Sample Manga', direction: 'ltr' }));
|
||||
await fsp.writeFile(path.join(mangaDir, 'cover.png'), PNG);
|
||||
await fsp.writeFile(path.join(chapterDir, 'chapter.json'), JSON.stringify({ title: 'The first chapter', source: { name: 'Local' } }));
|
||||
await fsp.writeFile(path.join(chapterDir, 'chapter.json'), JSON.stringify({ title: 'The first chapter', source: { name: 'Local' },
|
||||
added: '2026-10-01T08:00:00.000Z', 'x-dates': { version: 1, source: { at: '2026-09-30T15:00:00.000Z', name: 'WeebCentral', batch: false } } }));
|
||||
await fsp.writeFile(path.join(chapterDir, '0001.png'), PNG);
|
||||
const ga = await gateway(dataA, libA, portA, path.join(root, 'usage-a.json'));
|
||||
let gb = await gateway(dataB, libB, portB, path.join(root, 'usage-b.json'));
|
||||
@@ -134,7 +135,13 @@ test('mTLS pairing, opt-in catalog, peer queue publication, and revocation', { t
|
||||
assert.equal(result.publishedCount, 1);
|
||||
assert.deepEqual(JSON.parse(await fsp.readFile(path.join(libB, 'Sample Manga', 'manga.json'), 'utf8'))['x-work-identity'], remote.workIdentity,
|
||||
'The stable work identity is stored with the imported peer edition');
|
||||
assert.equal(fs.existsSync(path.join(libB, 'Sample Manga', 'v01', 'c0001 The first chapter [Peer ServerA ' + inB.id.slice(1, 7) + ']', '0001.png')), true);
|
||||
const receivedDir = path.join(libB, 'Sample Manga', 'v01', 'c0001 The first chapter [Peer ServerA ' + inB.id.slice(1, 7) + ']');
|
||||
assert.equal(fs.existsSync(path.join(receivedDir, '0001.png')), true);
|
||||
const received = JSON.parse(await fsp.readFile(path.join(receivedDir, 'chapter.json'), 'utf8'));
|
||||
assert.deepEqual(received['x-dates'].ecosystem, { at: '2026-10-01T08:00:00.000Z', via: 'peer ServerA' },
|
||||
'The chapter keeps the date it first entered Manga Circus on A, not its arrival on B');
|
||||
assert.equal(received['x-dates'].source.at, '2026-09-30T15:00:00.000Z');
|
||||
assert.notEqual(received.added, '2026-10-01T08:00:00.000Z', '"added" is the arrival in B\'s own library');
|
||||
const second = await run(info, { scope: { mode: 'missing' }, existingPolicy: 'skip-any' },
|
||||
{ libraryRoot: libB, settings: { chapterMinMs: 500 }, shouldStop: () => false });
|
||||
assert.equal(second.publishedCount, 0);
|
||||
|
||||
@@ -180,6 +180,11 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; }
|
||||
.ch { display: flex; width: 100%; max-width: 100%; align-items: center; gap: 6px; border-bottom: 1px solid var(--line); white-space: nowrap; }
|
||||
.ch-main { flex: 1 1 auto; min-width: 0; display: grid; grid-template-columns: 76px 52px minmax(70px, 1fr) max-content; gap: 8px; align-items: baseline; padding: 6px 4px; text-decoration: none; }
|
||||
.ch-main.has-time { grid-template-columns: 76px 52px 72px minmax(70px, 1fr) max-content; }
|
||||
.ch-main.has-date { grid-template-columns: 76px 52px 104px minmax(70px, 1fr) max-content; }
|
||||
.ch-main.has-time.has-date { grid-template-columns: 76px 52px 104px 72px minmax(70px, 1fr) max-content; }
|
||||
.ch-date { min-width: 0; overflow: hidden; color: var(--muted); font-size: 13px; font-variant-numeric: tabular-nums; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.ch-date.tier-official { color: inherit; }
|
||||
.ch-date.tier-ecosystem, .ch-date.tier-library { opacity: .75; }
|
||||
.ch-main:hover .ch-title { text-decoration: underline; }
|
||||
.ch-label { min-width: 0; overflow: hidden; text-overflow: ellipsis; font-weight: 650; font-variant-numeric: tabular-nums; }
|
||||
.ch-pages { min-width: 0; overflow: hidden; color: var(--muted); font-size: 13px; font-variant-numeric: tabular-nums; text-align: left; white-space: nowrap; }
|
||||
@@ -205,6 +210,10 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; }
|
||||
.ch { flex-wrap: nowrap; gap: 4px; overflow: hidden; }
|
||||
.ch-main { grid-template-columns: 64px 44px minmax(0, 1fr) max-content; gap: 6px; }
|
||||
.ch-main.has-time { grid-template-columns: 64px 44px 62px minmax(0, 1fr) max-content; }
|
||||
/* Phones: the date takes the page-count column. */
|
||||
.ch-main.has-date { grid-template-columns: 64px 64px minmax(0, 1fr) max-content; }
|
||||
.ch-main.has-time.has-date { grid-template-columns: 64px 64px 62px minmax(0, 1fr) max-content; }
|
||||
.ch-main.has-date .ch-pages { display: none; }
|
||||
.ch > .btn.small { min-width: 0; padding-left: 5px; padding-right: 5px; }
|
||||
.ch > .state { flex: none; }
|
||||
}
|
||||
|
||||
+45
-5
@@ -213,14 +213,16 @@
|
||||
const panel = h('div', { class: 'versions', hidden: true });
|
||||
function draw() {
|
||||
const p = ch.progress, done = p && p.done;
|
||||
const state = h('button', { class: 'state' + (done ? ' done' : ''), title: done ? 'Mark as unread' : 'Mark as read' },
|
||||
done ? '✓ read' : p ? `page ${p.p + 1} / ${p.n}` : 'unread');
|
||||
const readAt = done ? readDate(ch.readAt) : null;
|
||||
const state = h('button', { class: 'state' + (done ? ' done' : ''), title: (readAt ? readAt.title + ' · ' : '') + (done ? 'Mark as unread' : 'Mark as read') },
|
||||
done ? (readAt ? '✓ ' + readAt.text : '✓ read') : p ? `page ${p.p + 1} / ${p.n}` : 'unread');
|
||||
state.onclick = async () => {
|
||||
if (done) { await api('api/unread', { m: M.id, c: ch.id }); ch.progress = null; }
|
||||
if (done) { await api('api/unread', { m: M.id, c: ch.id }); ch.progress = null; ch.readAt = null; }
|
||||
else {
|
||||
const version = ch.versions[0];
|
||||
await api('api/progress', { m: M.id, c: ch.id, v: version.id, p: version.pages - 1, n: version.pages, done: true });
|
||||
ch.progress = { v: version.id, p: version.pages - 1, n: version.pages, done: true };
|
||||
ch.readAt = { at: Date.now(), approx: false };
|
||||
}
|
||||
draw();
|
||||
};
|
||||
@@ -228,12 +230,14 @@
|
||||
? h('button', { class: 'btn small', onclick: () => { panel.hidden = !panel.hidden; } }, ch.versions.length + ' versions')
|
||||
: null;
|
||||
const pages = ch.versions[0] ? ch.versions[0].pages : 0;
|
||||
const timing = chapterTiming(ch);
|
||||
const timing = chapterTiming(ch), released = chapterRelease(ch.dates);
|
||||
wrap.textContent = '';
|
||||
wrap.append(h('div', { class: 'ch' + (done ? ' read' : '') },
|
||||
h('a', { class: 'ch-main' + (timing ? ' has-time' : ''), href: readUrl(M.id, ch.id) },
|
||||
h('a', { class: 'ch-main' + (timing ? ' has-time' : '') + (M.chapterDates?.release ? ' has-date' : ''), href: readUrl(M.id, ch.id) },
|
||||
h('span', { class: 'ch-label' }, ch.label),
|
||||
h('span', { class: 'ch-pages' }, `${pages} p.`),
|
||||
M.chapterDates?.release ? h('span', { class: 'ch-date' + (released ? ' tier-' + released.tier : ''), title: released ? released.title : 'No date known', 'aria-label': released ? released.title : 'No date known' },
|
||||
released ? released.text : '–') : null,
|
||||
timing ? h('span', { class: 'ch-time', title: timing.title, 'aria-label': timing.title }, timing.text) : null,
|
||||
h('span', { class: 'ch-title', title: ch.title || '' }, ch.title || 'Untitled'),
|
||||
ch.extra ? h('span', { class: 'tag' }, 'extra') : null),
|
||||
@@ -261,6 +265,42 @@
|
||||
return wrap;
|
||||
}
|
||||
|
||||
// Phones (the chapter-row breakpoint in app.css) get numeric dates to leave room for titles.
|
||||
const compact = matchMedia('(max-width: 640px)').matches;
|
||||
|
||||
// Read date: the day and the hour (local time) the chapter was first completed.
|
||||
function readDate(value) {
|
||||
const mode = M.chapterDates?.read;
|
||||
if (!value || !value.at || mode === 'off') return null;
|
||||
const when = new Date(value.at), thisYear = when.getFullYear() === new Date().getFullYear();
|
||||
const day = when.toLocaleDateString(undefined, compact ? { day: '2-digit', month: '2-digit', ...(thisYear ? {} : { year: '2-digit' }) }
|
||||
: { day: 'numeric', month: 'short', ...(thisYear ? {} : { year: 'numeric' }) });
|
||||
const text = (value.approx ? '≤ ' : '') + day + (mode === 'hour' ? `, ${String(when.getHours()).padStart(2, '0')}h` : '');
|
||||
const full = when.toLocaleDateString(undefined, { dateStyle: 'long' }) + (mode === 'hour' ? `, ${String(when.getHours()).padStart(2, '0')}:00–${String((when.getHours() + 1) % 24).padStart(2, '0')}:00` : '');
|
||||
return { text, title: (value.approx ? 'Read by ' : 'Read on ') + full + (value.approx ? ' (earlier reads were not recorded)' : '') };
|
||||
}
|
||||
|
||||
// Release date by tier: official > source upload > Manga Circus ecosystem > this library.
|
||||
const TIER = { official: { mark: '' }, source: { mark: '≈' }, ecosystem: { mark: '◎' }, library: { mark: '↓' } };
|
||||
const dayText = (at, precision, long) => {
|
||||
// Day-precision dates are calendar dates, not instants: show them in UTC so they do not shift.
|
||||
const dayOnly = !String(at).includes('T');
|
||||
const opts = precision === 'year' ? { year: 'numeric' } : precision === 'month' ? { year: 'numeric', month: 'short' }
|
||||
: long ? { dateStyle: 'long' } : compact ? { day: '2-digit', month: '2-digit', year: '2-digit' } : { day: 'numeric', month: 'short', year: 'numeric' };
|
||||
return new Date(at).toLocaleDateString(undefined, { ...opts, ...(dayOnly ? { timeZone: 'UTC' } : {}) });
|
||||
};
|
||||
function chapterRelease(dates) {
|
||||
if (!dates || !dates.release) return null;
|
||||
const known = dates.known || {}, lines = [];
|
||||
if (known.official) lines.push(`Released ${dayText(known.official.at, known.official.precision, true)}${known.official.name ? ' (' + known.official.name + ')' : ''}`);
|
||||
if (known.source) lines.push(`Uploaded at ${known.source.name || 'the source'} ${dayText(known.source.at, 'day', true)}`);
|
||||
if (known.batch) lines.push(`Bulk upload at ${known.batch.name || 'the source'} ${dayText(known.batch.at, 'day', true)} (not a release date)`);
|
||||
if (known.ecosystem) lines.push(`First in Manga Circus ${dayText(known.ecosystem.at, 'day', true)}${known.ecosystem.name && known.ecosystem.name !== 'this server' ? ' (via ' + known.ecosystem.name + ')' : ''}`);
|
||||
if (known.library) lines.push(`Added to this library ${dayText(known.library.at, 'day', true)}`);
|
||||
const tier = TIER[dates.tier] || TIER.library;
|
||||
return { tier: dates.tier, text: tier.mark + dayText(dates.release.at, dates.release.precision), title: lines.join('\n') };
|
||||
}
|
||||
|
||||
function chapterTiming(chapter) {
|
||||
if (!M.reading) return null;
|
||||
const value = chapter.reading;
|
||||
|
||||
@@ -38,6 +38,11 @@
|
||||
h('option', { value: 'dim' }, 'Dim (70% brightness)'),
|
||||
h('option', { value: 'extra-dim' }, 'Extra dim (45% brightness)'));
|
||||
tone.value = S.pageTone;
|
||||
const readDate = h('select', { onchange: e => save({ readDate: e.target.value }) },
|
||||
h('option', { value: 'hour' }, 'Date and hour (e.g. 23 Oct, 21h)'),
|
||||
h('option', { value: 'date' }, 'Date only'),
|
||||
h('option', { value: 'off' }, 'Do not show'));
|
||||
readDate.value = S.readDate;
|
||||
const LABEL = { flow: 'Flow', page: 'Page', book: 'Book', strip: 'Webtoon flow' };
|
||||
const zooms = () => Object.keys(LABEL).map(k => `${LABEL[k]} ${Math.round((S.zoom[k] || 1) * 100)}%`).join(' · ');
|
||||
const zoomText = h('span', { class: 'muted' }, zooms());
|
||||
@@ -85,6 +90,13 @@
|
||||
h('button', { class: 'btn small', onclick: async () => { await save({ zoom: {} }); zoomText.textContent = zooms(); } }, 'Reset to fit'),
|
||||
h('div', { class: 'muted small' }, '100% = the page fits the screen. Change it in the reader with Ctrl + mouse wheel, a pinch, or the + / - keys.'))));
|
||||
|
||||
section('user/reading').append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Chapter list dates'),
|
||||
h('label', { class: 'field' }, h('span', {}, 'When I read a chapter'), readDate,
|
||||
h('div', { class: 'muted small' }, 'Shown next to ✓ in the chapter list: when you first finished the chapter, in your local time. Marking it unread clears it.')),
|
||||
check('releaseDate', 'Show when each chapter was released',
|
||||
'Best known date first: the official release, else the upload at the source (≈), else when it first reached Manga Circus (◎), else when it was added here (↓). Hover a date for all known dates.')));
|
||||
|
||||
const readingHoursBody = h('div', {}, h('p', { class: 'muted' }, 'Loading server reading hours…'));
|
||||
section('user/reading-hours').append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Server reading hours'), readingHoursBody,
|
||||
|
||||
+12
-1
@@ -115,6 +115,17 @@ function chapterInsight(record) {
|
||||
};
|
||||
}
|
||||
|
||||
// Earliest completion time the clock still remembers (only the last few visits are kept).
|
||||
// approx: older completed passes were already dropped, so the first read may be earlier.
|
||||
function earliestCompletion(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
const passes = [record.currentPass, ...(record.recentSessions || []).map(item => item && item.pass)];
|
||||
const times = passes.map(pass => pass && whole(pass.completedAt)).filter(time => time > 0);
|
||||
if (!times.length) return null;
|
||||
const counted = new Set(times).size;
|
||||
return { at: Math.min(...times), approx: whole(record.completedPasses) > counted };
|
||||
}
|
||||
|
||||
function median(values) {
|
||||
const sorted = values.filter(value => Number.isFinite(value) && value > 0).sort((a, b) => a - b);
|
||||
if (!sorted.length) return null;
|
||||
@@ -295,6 +306,6 @@ function applyPulse(record, pulse, { version, pageCount, eligible, now = Date.no
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
INPUT_TYPES, applyPulse, applyVisit, chapterInsight, compactRanges, eligiblePages, emptyRecord,
|
||||
INPUT_TYPES, applyPulse, applyVisit, chapterInsight, compactRanges, earliestCompletion, eligiblePages, emptyRecord,
|
||||
libraryInsight, mangaInsight, publicSummary, rangeSet,
|
||||
};
|
||||
@@ -3,6 +3,7 @@
|
||||
const API = 'https://api.mangadex.org';
|
||||
const { parse } = require('../plugins/registry');
|
||||
const { localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary, pageUrls } = require('../plugins/mangadex');
|
||||
const { sourceDate } = require('../../chapter-dates');
|
||||
|
||||
function mangaDexInfo(value) {
|
||||
let url;
|
||||
@@ -71,8 +72,9 @@ async function discoverMangaDexChapter(ref, http) {
|
||||
const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`);
|
||||
const groupName = ref.group?.name ? ` · ${ref.group.name}` : '';
|
||||
const parsed = parse('mangadex', 'atHome', { data: { response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }, url: `${API}/at-home/server/${ref.id}` });
|
||||
const source = sourceDate(ref.uploadedAt, `MangaDex${groupName}`, ref.uploadBatch);
|
||||
return {
|
||||
number: ref.number, title: ref.title || '', added: ref.added,
|
||||
number: ref.number, title: ref.title || '', ...(source ? { dates: { source } } : {}),
|
||||
...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}),
|
||||
source: { name: `MangaDex${groupName}`, url: ref.url },
|
||||
pages: parsed.pages,
|
||||
|
||||
@@ -4,6 +4,7 @@ const { SlowHttp } = require('../lib/http');
|
||||
const { fallbackDirection } = require('../../reading-direction');
|
||||
const parser = require('../plugins/weebcentral');
|
||||
const { parse } = require('../plugins/registry');
|
||||
const { sourceDate } = require('../../chapter-dates');
|
||||
|
||||
function weebCentralInfo(value) {
|
||||
let url;
|
||||
@@ -36,7 +37,8 @@ async function discoverWeebCatalog(seriesUrl, http) {
|
||||
|
||||
async function discoverWeebChapter(chapter, http) {
|
||||
const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text();
|
||||
const { manifestUrl, added } = parse('weebcentral', 'chapter', { html: pageHtml, url: chapter.url });
|
||||
const { manifestUrl } = parse('weebcentral', 'chapter', { html: pageHtml, url: chapter.url });
|
||||
const source = sourceDate(chapter.uploadedAt, 'WeebCentral', chapter.uploadBatch);
|
||||
const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip¤t_page=1`, { accept: 'text/html' })).text();
|
||||
const parsed = parse('weebcentral', 'pages', { html: fragment, url: manifestUrl });
|
||||
const pages = parsed.pages.map(url => ({ url, headers: { referer: chapter.url } }));
|
||||
@@ -46,7 +48,7 @@ async function discoverWeebChapter(chapter, http) {
|
||||
...(chapter.special ? { special: true, label: chapter.label, sourceId: chapter.id,
|
||||
sourceNumber: chapter.volume, releaseKind: chapter.releaseKind, volume: chapter.volume } : {}),
|
||||
...(chapter.book ? { book: chapter.book } : {}),
|
||||
added,
|
||||
...(source ? { dates: { source } } : {}),
|
||||
source: { name: 'WeebCentral', url: chapter.url }, pages,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ const { directionFields } = require('../../reading-direction');
|
||||
const { mergeImportedMetadata } = require('./metadata-merge');
|
||||
const { normalizeWorkIdentity } = require('./work-identity');
|
||||
const { parseKey } = require('../../chapter-identity');
|
||||
const { publishedDates } = require('../../chapter-dates');
|
||||
|
||||
const token = () => crypto.randomBytes(5).toString('hex');
|
||||
// Staging is a recovery asset, not disposable garbage. Legacy stages without an
|
||||
@@ -180,7 +181,8 @@ async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) {
|
||||
await fsp.writeFile(path.join(stage, 'chapter.json'), JSON.stringify({
|
||||
...priorMeta,
|
||||
title: chapter.title || '', source: chapter.source,
|
||||
added: chapter.added || new Date().toISOString(),
|
||||
// "added" stays the first arrival in this library; x-dates keeps source/release/ecosystem.
|
||||
...publishedDates(priorMeta, { ...(chapter.dates || {}), added: chapter.added }),
|
||||
priority: Number(chapter.priority || 0), extra: !!chapter.extra,
|
||||
...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId,
|
||||
sourceNumber: chapter.sourceNumber ?? null,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const { specialKey } = require('../../chapter-identity');
|
||||
const { batchFlags } = require('../../chapter-dates');
|
||||
|
||||
function localizedTitle(attributes = {}) {
|
||||
const titles = attributes.title || {};
|
||||
@@ -41,14 +42,18 @@ function selectChapterReleases(resources = [], specialLabel = 'Special') {
|
||||
const prior = selected.get(number);
|
||||
if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource);
|
||||
}
|
||||
return [...selected.entries()].map(([number, resource]) => {
|
||||
const chosen = [...selected.entries()];
|
||||
const uploaded = chosen.map(([, resource]) => resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt);
|
||||
const batches = batchFlags(uploaded);
|
||||
return chosen.map(([number, resource], index) => {
|
||||
const group = groupFor(resource);
|
||||
return {
|
||||
id: resource.id, number, title: resource.attributes?.title || '',
|
||||
...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}),
|
||||
...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}),
|
||||
pages: Number(resource.attributes?.pages || 0),
|
||||
added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt,
|
||||
// A group's upload time, not an official release; bulk uploads are flagged.
|
||||
...(uploaded[index] ? { uploadedAt: uploaded[index], uploadBatch: !!batches[index] } : {}),
|
||||
group,
|
||||
url: `https://mangadex.org/chapter/${resource.id}`,
|
||||
};
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
'use strict';
|
||||
const { volumeKey } = require('../../chapter-identity');
|
||||
const { batchFlags } = require('../../chapter-dates');
|
||||
|
||||
// First-party, pure parsing functions. This module never fetches remote content.
|
||||
const decode = value => value.replace(/&/g, '&').replace(/'/g, "'").replace(/"/g, '"').replace(/<[^>]+>/g, '').trim();
|
||||
@@ -21,19 +22,26 @@ function catalog({ html, url }) {
|
||||
const volume = !regular ? text.match(/^Volume\s+([0-9]+(?:\.[0-9]+)?)(?!\d)/i)?.[1] : null;
|
||||
const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1];
|
||||
const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null);
|
||||
// The full chapter list carries each upload time; the chapter page itself has none.
|
||||
const uploaded = match[3].match(/datetime="([^"]+)"/i)?.[1];
|
||||
const uploadedAt = uploaded && Number.isFinite(Date.parse(uploaded)) ? uploaded : undefined;
|
||||
if (number) found.push({
|
||||
id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(),
|
||||
...(season ? { book: { number: season, title: `Season ${season}` } } : {}),
|
||||
url: absolute(url, match[1]), raw: text,
|
||||
url: absolute(url, match[1]), raw: text, ...(uploadedAt ? { uploadedAt } : {}),
|
||||
});
|
||||
else if (volume) found.push({
|
||||
id: match[2], number: volumeKey(match[2]), title: '', special: true,
|
||||
label: `Volume ${volume}`, releaseKind: 'volume', volume,
|
||||
book: { number: Number(volume), title: `Volume ${volume}` },
|
||||
url: absolute(url, match[1]), raw: text,
|
||||
url: absolute(url, match[1]), raw: text, ...(uploadedAt ? { uploadedAt } : {}),
|
||||
});
|
||||
}
|
||||
const chapters = [...new Map(found.map(chapter => [chapter.id, chapter])).values()];
|
||||
// Back-catalog chapters were imported together; their upload time is not a release date.
|
||||
batchFlags(chapters.map(chapter => chapter.uploadedAt)).forEach((batch, index) => {
|
||||
if (batch !== null) chapters[index].uploadBatch = batch;
|
||||
});
|
||||
const chapterLinks = [...html.matchAll(/<a\b[^>]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length;
|
||||
return { chapters, chapterLinks };
|
||||
}
|
||||
@@ -42,7 +50,6 @@ function chapter({ html, url }) {
|
||||
return {
|
||||
manifestUrl: html.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1]
|
||||
|| `${url}/images?is_prev=False`,
|
||||
added: html.match(/datetime="([^"]+)"/i)?.[1],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ const { adapters, adapterFor } = require('../adapters');
|
||||
const seriesUrl = 'https://weebcentral.com/series/ABC/title';
|
||||
const chapterUrl = 'https://weebcentral.com/chapters/XYZ';
|
||||
const seriesHtml = '<h1>Test Manga</h1><div hx-get="/series/ABC/full-chapter-list"></div>';
|
||||
const listHtml = '<a href="/chapters/XYZ">Chapter 1</a><a href="/chapters/LMN">Log. 2</a>';
|
||||
const chapterHtml = '<time datetime="2026-01-01"></time>';
|
||||
const listHtml = '<a href="/chapters/XYZ"><span>Chapter 1</span><time datetime="2026-01-01T15:06:05.348Z">x</time></a><a href="/chapters/LMN">Log. 2</a>';
|
||||
const chapterHtml = '<div>no dates on the chapter page</div>';
|
||||
const imagesHtml = '<img src="https://img.example/1.jpg"><img data-src="https://img.example/2.jpg">';
|
||||
|
||||
test('first-party registry rejects unknown plugins and malformed input', () => {
|
||||
@@ -45,7 +45,9 @@ test('in-process parser preserves WeebCentral catalog and page discovery', async
|
||||
assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']);
|
||||
const chapter = await discoverWeebChapter(catalog.chapters[0], http);
|
||||
assert.equal(chapter.pages.length, 2);
|
||||
assert.equal(chapter.added, '2026-01-01');
|
||||
assert.equal(chapter.added, undefined);
|
||||
assert.deepEqual(chapter.dates, { source: { at: '2026-01-01T15:06:05.348Z', name: 'WeebCentral', batch: false } });
|
||||
assert.equal(catalog.chapters[1].uploadedAt, undefined);
|
||||
assert.equal(requested.length, 4);
|
||||
});
|
||||
|
||||
|
||||
@@ -25,8 +25,9 @@ const peerClient = require('./peering/client');
|
||||
const { DiscoveryStore } = require('./discovery/store');
|
||||
const { generate: generateDiscovery } = require('./discovery/engine');
|
||||
const {
|
||||
applyPulse, applyVisit, chapterInsight, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary,
|
||||
applyPulse, applyVisit, chapterInsight, earliestCompletion, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary,
|
||||
} = require('./reading-analytics');
|
||||
const { chapterDates } = require('./chapter-dates');
|
||||
const { parseSchedule, publicSchedule, loadQuestionBank, createReadingAccess } = require('./reading-access');
|
||||
const { BackupService, BackupError } = require('./backup');
|
||||
|
||||
@@ -97,7 +98,7 @@ function checkPw(pw, stored) {
|
||||
}
|
||||
const DUMMY_HASH = hashPw('x');
|
||||
const DEFAULTS = { startMode: 'last', lastMode: 'page', adapt: true, skipExtras: false, skipCovers: false, flowPageGap: true, pageTone: 'original', readingClock: false, readingStory: true,
|
||||
discoveryEnabled: false, discoveryReadingActivity: false, discoveryMature: false, zoom: {}, webhook: '' };
|
||||
readDate: 'hour', releaseDate: true, discoveryEnabled: false, discoveryReadingActivity: false, discoveryMature: false, zoom: {}, webhook: '' };
|
||||
const settingsOf = u => ({ ...DEFAULTS, ...u.settings, zoom: { ...(u.settings && u.settings.zoom) } });
|
||||
|
||||
function createUser(name, pw, admin, replace) {
|
||||
@@ -322,6 +323,13 @@ function firstReleaseOf(meta) {
|
||||
}
|
||||
return { label: label || '', time: Number.isFinite(time) ? time : 0 };
|
||||
}
|
||||
// When a done chapter was completed: the progress record, or for reads saved before doneAt
|
||||
// existed, the earliest completion the reading clock still remembers.
|
||||
function readAtOf(progress, record) {
|
||||
if (!progress || !progress.done) return null;
|
||||
if (progress.doneAt) return { at: progress.doneAt, approx: false };
|
||||
return earliestCompletion(record);
|
||||
}
|
||||
function continueOf(m, prog, s) {
|
||||
const ok = c => !(s.skipExtras && c.extra);
|
||||
let last = null;
|
||||
@@ -437,7 +445,8 @@ const routes = {
|
||||
if (['last', 'flow', 'page', 'book'].includes(b.startMode)) s.startMode = b.startMode;
|
||||
if (['flow', 'page', 'book'].includes(b.lastMode)) s.lastMode = b.lastMode;
|
||||
if (['original', 'inverted', 'sepia', 'dim', 'extra-dim'].includes(b.pageTone)) s.pageTone = b.pageTone;
|
||||
for (const k of ['adapt', 'skipExtras', 'skipCovers', 'flowPageGap', 'readingClock', 'readingStory',
|
||||
if (['off', 'date', 'hour'].includes(b.readDate)) s.readDate = b.readDate;
|
||||
for (const k of ['adapt', 'skipExtras', 'skipCovers', 'flowPageGap', 'readingClock', 'readingStory', 'releaseDate',
|
||||
'discoveryEnabled', 'discoveryReadingActivity', 'discoveryMature']) if (typeof b[k] === 'boolean') s[k] = b[k];
|
||||
if (b.zoom && typeof b.zoom === 'object') {
|
||||
s.zoom = {};
|
||||
@@ -557,9 +566,10 @@ const routes = {
|
||||
},
|
||||
|
||||
'GET /api/manga': ({ u, q }) => {
|
||||
const m = mangaOf(q.get('m')), prog = progressOf(u, m), showStory = settingsOf(u).readingStory;
|
||||
const m = mangaOf(q.get('m')), prog = progressOf(u, m), s = settingsOf(u), showStory = s.readingStory;
|
||||
const records = mangaReadingOf(u, m);
|
||||
return {
|
||||
chapterDates: { read: s.readDate, release: s.releaseDate },
|
||||
id: m.id, title: m.title, meta: m.meta, cover: url(m.cover), direction: m.direction,
|
||||
collections: catalog.state.collections.filter(c => c.members.some(member => member.mangaId === m.id)).map(c => ({ id: c.id, title: c.title })),
|
||||
relations: catalog.state.relations.filter(r => r.state === 'confirmed' && (r.from === m.id || r.to === m.id)).map(r => ({ ...r,
|
||||
@@ -569,6 +579,8 @@ const routes = {
|
||||
chapters: m.chapters.map(ch => ({
|
||||
id: ch.id, book: ch.book, label: ch.label, title: ch.title, extra: ch.extra, progress: prog[ch.id] || null, versions: versionsOf(m, ch, u),
|
||||
reading: showStory ? chapterInsight(records[ch.id]) : null,
|
||||
readAt: s.readDate === 'off' ? null : readAtOf(prog[ch.id], records[ch.id]),
|
||||
dates: s.releaseDate ? chapterDates(ch.versions) : null,
|
||||
})),
|
||||
reading: showStory ? mangaInsight(records, m.chapters.map(ch => ch.id)) : null,
|
||||
continue: continueOf(m, prog, settingsOf(u)),
|
||||
@@ -637,8 +649,11 @@ const routes = {
|
||||
'POST /api/progress': ({ u, b }) => {
|
||||
const m = mangaOf(b.m), ch = chapterOf(m, b.c);
|
||||
const all = (db.progress[u.name] = db.progress[u.name] || {}), mp = (all[m.id] = all[m.id] || {}), old = mp[ch.id] || {};
|
||||
const now = Date.now(), done = !!(old.done || b.done);
|
||||
// doneAt: when the chapter was first completed; kept until it is marked unread.
|
||||
const doneAt = old.done ? old.doneAt : done ? now : undefined;
|
||||
mp[ch.id] = {
|
||||
v: String(b.v || ''), p: Math.max(0, b.p | 0), n: Math.max(0, b.n | 0), t: Date.now(), done: !!(old.done || b.done),
|
||||
v: String(b.v || ''), p: Math.max(0, b.p | 0), n: Math.max(0, b.n | 0), t: now, done, ...(doneAt ? { doneAt } : {}),
|
||||
shift: Array.isArray(b.shift) ? b.shift.filter(Number.isInteger).slice(0, 1000) : old.shift || [],
|
||||
};
|
||||
save();
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { batchFlags, publishedDates, chapterDates, peerDates, fromPeer } = require('../chapter-dates');
|
||||
const { earliestCompletion } = require('../reading-analytics');
|
||||
|
||||
test('bulk imports are flagged, weekly uploads are not', () => {
|
||||
const flags = batchFlags([
|
||||
'2024-09-07T10:00:00Z', '2024-09-07T10:00:03Z', '2024-09-07T10:04:00Z', // one import
|
||||
'2026-10-11T15:05:58Z', '2026-10-18T15:06:05Z', // weekly releases
|
||||
'2026-10-20T09:00:00Z', '2026-10-20T09:01:00Z', // a double release stays a release
|
||||
undefined,
|
||||
]);
|
||||
assert.deepEqual(flags, [true, true, true, false, false, false, false, null]);
|
||||
});
|
||||
|
||||
test('publishing keeps the first arrival and the earliest ecosystem date', () => {
|
||||
const now = Date.parse('2026-10-24T12:00:00Z');
|
||||
const fresh = publishedDates({}, { source: { at: '2026-10-18T15:06:05Z', name: 'WeebCentral', batch: false } }, now);
|
||||
assert.equal(fresh.added, '2026-10-24T12:00:00.000Z');
|
||||
assert.deepEqual(fresh['x-dates'].ecosystem, { at: '2026-10-24T12:00:00.000Z', via: 'this server' });
|
||||
assert.equal(fresh['x-dates'].source.name, 'WeebCentral');
|
||||
|
||||
// Replacing a version later: "added" and the ecosystem date survive.
|
||||
const replaced = publishedDates(fresh, {}, now + 864e5);
|
||||
assert.equal(replaced.added, fresh.added);
|
||||
assert.deepEqual(replaced['x-dates'].ecosystem, fresh['x-dates'].ecosystem);
|
||||
assert.equal(replaced['x-dates'].source.at, '2026-10-18T15:06:05.000Z');
|
||||
|
||||
// A peer that had it earlier lowers the date; a later claim never raises it.
|
||||
const earlier = publishedDates(fresh, { ecosystem: { at: '2026-10-19T08:00:00Z', via: 'peer bob' } }, now);
|
||||
assert.deepEqual(earlier['x-dates'].ecosystem, { at: '2026-10-19T08:00:00.000Z', via: 'peer bob' });
|
||||
const later = publishedDates(earlier, { ecosystem: { at: '2026-10-23T00:00:00Z', via: 'peer eve' } }, now);
|
||||
assert.equal(later['x-dates'].ecosystem.via, 'peer bob');
|
||||
});
|
||||
|
||||
test('peer ecosystem claims cannot predate the source upload or lie in the future', () => {
|
||||
const now = Date.parse('2026-10-24T12:00:00Z');
|
||||
const source = { at: '2026-10-18T15:06:05Z', name: 'WeebCentral', batch: false };
|
||||
const early = publishedDates({}, { source, ecosystem: { at: '2001-01-01T00:00:00Z', via: 'peer x' } }, now);
|
||||
assert.equal(early['x-dates'].ecosystem.at, '2026-10-18T15:06:05.000Z');
|
||||
const future = publishedDates({}, { ecosystem: { at: '2099-01-01T00:00:00Z', via: 'peer x' } }, now);
|
||||
assert.equal(future['x-dates'].ecosystem.at, '2026-10-24T12:00:00.000Z');
|
||||
});
|
||||
|
||||
test('release date tiers: official, source upload, ecosystem, library', () => {
|
||||
const added = Date.parse('2026-10-17T15:04:03Z');
|
||||
const version = dates => ({ added, meta: { 'x-dates': dates } });
|
||||
const ecosystem = { at: '2026-10-10T00:00:00Z', via: 'peer bob' };
|
||||
assert.equal(chapterDates([version({})]).tier, 'library');
|
||||
assert.equal(chapterDates([version({ ecosystem })]).release.at, '2026-10-10T00:00:00.000Z');
|
||||
const batch = chapterDates([version({ ecosystem, source: { at: '2024-09-07T10:00:00Z', name: 'WeebCentral', batch: true } })]);
|
||||
assert.equal(batch.tier, 'ecosystem', 'a bulk upload is no release date');
|
||||
assert.equal(batch.known.batch.at, '2024-09-07T10:00:00.000Z');
|
||||
const source = chapterDates([version({ ecosystem }), version({ source: { at: '2026-10-04T15:06:05Z', name: 'MangaDex', batch: false } })]);
|
||||
assert.equal(source.tier, 'source');
|
||||
assert.equal(source.known.ecosystem.at, '2026-10-10T00:00:00.000Z', 'earliest across versions');
|
||||
const official = chapterDates([version({ release: { at: '2023-09-17', precision: 'day', source: 'MangaUpdates · Viz' }, ecosystem })]);
|
||||
assert.deepEqual(official.release, { at: '2023-09-17', precision: 'day', name: 'MangaUpdates · Viz' });
|
||||
assert.equal(official.tier, 'official');
|
||||
});
|
||||
|
||||
test('peers exchange cleaned dates and the receiver records the peer', () => {
|
||||
const meta = { added: '2026-10-17T15:04:03Z', secret: 'x',
|
||||
'x-dates': { source: { at: '2026-10-18T15:06:05Z', name: 'WeebCentral', batch: false, extra: 'drop' } } };
|
||||
const sent = peerDates(meta, 0);
|
||||
assert.deepEqual(sent, { ecosystem: '2026-10-17T15:04:03.000Z', source: { at: '2026-10-18T15:06:05.000Z', name: 'WeebCentral', batch: false } });
|
||||
const received = fromPeer(sent, 'bob');
|
||||
assert.deepEqual(received.ecosystem, { at: '2026-10-17T15:04:03.000Z', via: 'peer bob' });
|
||||
assert.deepEqual(fromPeer('nonsense', 'bob'), {});
|
||||
assert.deepEqual(fromPeer({ ecosystem: 'not a date' }, 'bob'), {});
|
||||
});
|
||||
|
||||
test('legacy reads get the earliest completion the reading clock remembers', () => {
|
||||
assert.equal(earliestCompletion(null), null);
|
||||
const record = { completedPasses: 3, currentPass: { completedAt: 5000 },
|
||||
recentSessions: [{ pass: { completedAt: 3000 } }, { pass: { completedAt: null } }, { pass: { completedAt: 5000 } }] };
|
||||
assert.deepEqual(earliestCompletion(record), { at: 3000, approx: true });
|
||||
assert.deepEqual(earliestCompletion({ ...record, completedPasses: 2 }), { at: 3000, approx: false });
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// One-time migration to chapter dates (BL-0047). Dry run by default; apply only with the
|
||||
// scraper stopped (--apply --writers-stopped), then run a deep "Rescan" in Settings.
|
||||
// - WeebCentral: x-dates.source from the series' full chapter list (one request per series).
|
||||
// - MangaDex: chapters published before x-dates stored the group's publishAt in "added";
|
||||
// it moves to x-dates.source and "added" becomes the folder's arrival time.
|
||||
// - Every chapter: x-dates.ecosystem = "added" unless an earlier one is already known.
|
||||
const fs = require('node:fs').promises;
|
||||
const path = require('node:path');
|
||||
const { batchFlags, sourceDate, publishedDates } = require('../chapter-dates');
|
||||
const weebcentral = require('../scraper/plugins/weebcentral');
|
||||
const { writeJsonAtomic } = require('../scraper/lib/publisher');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const apply = args.includes('--apply');
|
||||
const only = args.find(arg => arg.startsWith('--manga='))?.slice('--manga='.length);
|
||||
const offline = args.includes('--offline');
|
||||
const root = args.find(arg => !arg.startsWith('--'));
|
||||
if (!root || args.some(arg => arg.startsWith('--') && !['--apply', '--writers-stopped', '--offline'].includes(arg) && !arg.startsWith('--manga='))
|
||||
|| (apply && !args.includes('--writers-stopped'))) {
|
||||
console.error('Usage: node tools/backfill-chapter-dates.js <library> [--manga=<folder>] [--offline] [--apply --writers-stopped]');
|
||||
process.exit(2);
|
||||
}
|
||||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130 Safari/537.36';
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const readJson = async file => { try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; } };
|
||||
const dirs = async dir => (await fs.readdir(dir, { withFileTypes: true })).filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name).sort();
|
||||
|
||||
async function weebCentralUploads(seriesId) {
|
||||
const url = `https://weebcentral.com/series/${seriesId}/full-chapter-list`;
|
||||
const response = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, signal: AbortSignal.timeout(30000) });
|
||||
if (!response.ok) throw new Error(`WeebCentral HTTP ${response.status}`);
|
||||
const { chapters } = weebcentral.catalog({ html: await response.text(), url });
|
||||
return new Map(chapters.filter(c => c.uploadedAt).map(c => [c.id, sourceDate(c.uploadedAt, 'WeebCentral', c.uploadBatch)]));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const totals = { chapters: 0, changed: 0, alreadyDated: 0, weebSource: 0, weebBatch: 0, mangaDexMoved: 0, missing: 0 };
|
||||
for (const manga of await dirs(root)) {
|
||||
if (only && manga !== only) continue;
|
||||
const mangaDir = path.join(root, manga), meta = (await readJson(path.join(mangaDir, 'manga.json'))) || {};
|
||||
const versions = [];
|
||||
for (const book of await dirs(mangaDir)) {
|
||||
for (const chapter of await dirs(path.join(mangaDir, book))) {
|
||||
const dir = path.join(mangaDir, book, chapter), file = path.join(dir, 'chapter.json'), chapterMeta = await readJson(file);
|
||||
if (chapterMeta) versions.push({ dir, file, meta: chapterMeta, mtime: (await fs.stat(dir)).mtimeMs });
|
||||
}
|
||||
}
|
||||
let uploads = new Map();
|
||||
const seriesId = meta['x-scraper']?.ids?.weebcentral;
|
||||
if (seriesId && !offline && versions.some(v => /weebcentral\.com\/chapters\//.test(v.meta.source?.url || '') && !v.meta['x-dates'])) {
|
||||
try { uploads = await weebCentralUploads(seriesId); await sleep(2000); }
|
||||
catch (error) { console.warn(`[backfill] ${manga}: WeebCentral list failed: ${error.message}`); }
|
||||
}
|
||||
// Legacy MangaDex "added" values are group upload times; flag bulk uploads among them.
|
||||
const legacyDex = versions.filter(v => !v.meta['x-dates'] && /^MangaDex\b/.test(v.meta.source?.name || '') && Date.parse(v.meta.added));
|
||||
const dexBatch = new Map(batchFlags(legacyDex.map(v => v.meta.added)).map((flag, i) => [legacyDex[i], flag]));
|
||||
let changed = 0;
|
||||
for (const version of versions) {
|
||||
totals.chapters++;
|
||||
if (version.meta['x-dates']) { totals.alreadyDated++; continue; }
|
||||
const prior = { ...version.meta }, incoming = {};
|
||||
const weebId = /weebcentral\.com\/chapters\/([0-9A-Z]{26})/.exec(prior.source?.url || '')?.[1];
|
||||
if (weebId && uploads.has(weebId)) {
|
||||
incoming.source = uploads.get(weebId);
|
||||
totals[incoming.source.batch ? 'weebBatch' : 'weebSource']++;
|
||||
} else if (dexBatch.has(version)) {
|
||||
incoming.source = sourceDate(prior.added, prior.source.name, dexBatch.get(version));
|
||||
prior.added = new Date(version.mtime).toISOString();
|
||||
totals.mangaDexMoved++;
|
||||
} else totals.missing++;
|
||||
const next = { ...prior, ...publishedDates(prior, incoming, version.mtime) };
|
||||
changed++; totals.changed++;
|
||||
if (apply) await writeJsonAtomic(version.file, next);
|
||||
}
|
||||
if (changed) console.log(`[backfill] ${manga}: ${changed} of ${versions.length} chapter versions ${apply ? 'updated' : 'would change'}`);
|
||||
}
|
||||
console.log(JSON.stringify({ apply, ...totals }));
|
||||
if (apply) console.log('Run a deep "Rescan" in Settings so the reader picks up the edited chapter.json files.');
|
||||
}
|
||||
|
||||
main().catch(error => { console.error(error); process.exit(1); });
|
||||
Reference in new issue
Block a user