Consolidate MangaCircus app, scraper, and Compose baseline
This commit is contained in:
Maashkinoozhe committed
1 parent
cfb2d79103
commit
eef4ffa07c
103 files changed
+9311
-167
No files matched your search
+29
-5
@@ -1,5 +1,29 @@
|
||||
data
|
||||
library
|
||||
.claude
|
||||
.git
|
||||
node_modules
|
||||
# Explicit allowlist; runtime state and credentials never enter image layers.
|
||||
**
|
||||
!package.json
|
||||
!server.js
|
||||
!library.js
|
||||
!reading-direction.js
|
||||
!chapter-identity.js
|
||||
!catalog.js
|
||||
!reading-analytics.js
|
||||
!public/
|
||||
!public/**
|
||||
!runtime/
|
||||
!runtime/**
|
||||
!sentinel/
|
||||
!sentinel/**
|
||||
!scraper/
|
||||
!scraper/**
|
||||
!tools/
|
||||
!tools/**
|
||||
scraper/.cache/**
|
||||
scraper/logs/**
|
||||
scraper/*.manifest.json
|
||||
**/*.log
|
||||
**/*.tmp
|
||||
**/test/**
|
||||
**/node_modules/**
|
||||
**/.env*
|
||||
**/*.pem
|
||||
**/*.key
|
||||
@@ -0,0 +1,12 @@
|
||||
APP_IMAGE=mangacircus:local
|
||||
VCS_REF=development
|
||||
READER_BIND=127.0.0.1
|
||||
READER_PORT=8080
|
||||
LIBRARY_PATH=./library
|
||||
DATA_PATH=./data
|
||||
SCRAPER_DATA_PATH=./data/scraper
|
||||
SENTINEL_DATA_PATH=./data/sentinel
|
||||
SCRAPER_TOKEN_PATH=./.secrets/scraper_token
|
||||
TZ=Europe/Berlin
|
||||
COOKIE_SECURE=0
|
||||
PROXY_NETWORK=mangacircus-edge
|
||||
@@ -17,6 +17,9 @@ pnpm-debug.log*
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
!.env.*.example
|
||||
.secrets/
|
||||
/ops/proxmox/environment.json
|
||||
*.pem
|
||||
*.key
|
||||
|
||||
@@ -25,6 +28,9 @@ pnpm-debug.log*
|
||||
*.log
|
||||
.staging-*/
|
||||
.old-*/
|
||||
/scraper/.cache/
|
||||
/scraper/*.manifest.json
|
||||
/scraper/logs/
|
||||
|
||||
# --- Claude Code: keep the shared launch config, drop personal settings ---
|
||||
.claude/settings.local.json
|
||||
|
||||
+11
-6
@@ -1,12 +1,17 @@
|
||||
FROM node:24-alpine
|
||||
RUN apk add --no-cache tzdata
|
||||
WORKDIR /app
|
||||
COPY package.json server.js library.js ./
|
||||
ARG VCS_REF=development
|
||||
LABEL org.opencontainers.image.title="MangaCircus" org.opencontainers.image.revision=$VCS_REF
|
||||
COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js catalog.js ./
|
||||
COPY public ./public
|
||||
COPY tools ./tools
|
||||
RUN mkdir -p /library /data && chown node:node /data
|
||||
ENV PORT=8080 LIBRARY=/library DATA=/data
|
||||
VOLUME ["/data"]
|
||||
COPY scraper ./scraper
|
||||
COPY sentinel ./sentinel
|
||||
COPY runtime ./runtime
|
||||
RUN mkdir -p /library /data && chown node:node /data /library
|
||||
ENV PORT=8080 LIBRARY=/library DATA=/data TZ=Europe/Berlin
|
||||
EXPOSE 8080
|
||||
USER node
|
||||
HEALTHCHECK --interval=1m --timeout=5s CMD wget -qO- http://127.0.0.1:8080/api/auth >/dev/null || exit 1
|
||||
CMD ["node", "server.js"]
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=60s CMD node tools/healthcheck.js reader
|
||||
CMD ["node", "runtime/launch.js", "reader"]
|
||||
@@ -4,7 +4,7 @@ A no-frills, self-hosted manga / comic / webtoon reader.
|
||||
|
||||
- **Zero dependencies.** A Node.js server (`server.js`, `library.js`) and plain HTML/CSS/JS pages. No build step, no framework, no database server.
|
||||
- Reads the manga straight from a folder tree. The same folders stay readable in any normal picture viewer.
|
||||
- Accounts with per-user reading progress, view settings and "new chapter" notifications.
|
||||
- Accounts with per-user reading progress, view settings, an opt-in Active Reading Clock, a private Reading Story and "new chapter" notifications.
|
||||
- Reader with three view modes (**Flow**, **Page**, **Book**), a page zoom that leaves the controls alone, and tap zones for mobile.
|
||||
|
||||
## Run it
|
||||
@@ -19,13 +19,40 @@ Open http://localhost:8080. The first account you create becomes the admin. Try
|
||||
node tools/make-dummy.js library
|
||||
```
|
||||
|
||||
To run the reader, managed scraper and Sentinel together, use `npm run start:all`. Admins can
|
||||
then manage the persistent download queue, progress, pauses, retries, and rate limits
|
||||
under **Settings → Admin → Scraper**. The reader continues working if the scraper is stopped.
|
||||
**Settings → Admin → Collections & metadata** manages grouped tiles, reading order and relationships;
|
||||
**Settings → Admin → Service health** shows recovery incidents. See [collections](docs/collections.md)
|
||||
and [resilience](docs/resilience.md) for the ownership and agent-maintenance contracts.
|
||||
The current Windows installation uses Docker Compose; its old `MangaCircus Supervisor`
|
||||
Scheduled Task is disabled. Do not run a native duplicate writer. See
|
||||
[docs/local-deployment.md](docs/local-deployment.md) for its observed layout, health
|
||||
checks, restart procedure, and recovery point.
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
node tools/prepare-compose.js
|
||||
docker compose config -q
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
`docker-compose.yml` mounts `./library` read-only at `/library` and keeps accounts/progress in the `mangacircus-data` volume (`/data`).
|
||||
Stop existing native writers before using their data in containers. `docker-compose.yml`
|
||||
runs reader, scraper and Sentinel with explicit persistent bind mounts (`library/`,
|
||||
`data/`), file-backed secrets and only a loopback reader port. No named data volume is
|
||||
used. See [Compose deployment](docs/compose-deployment.md) for local development, VM
|
||||
recommendations, cutover, image releases and backup/rollback. The local Docker stack
|
||||
is verified; VM deployment is still pending.
|
||||
|
||||
### Settings workspace
|
||||
|
||||
Settings has **User** and **Admin** menu groups. User sections cover Reading,
|
||||
Reading time & privacy, Notifications and Account. Admin sections cover Scraper,
|
||||
Server & library, Users & access, Collections & metadata and Service health.
|
||||
Scraper has local Queue, Automatic updates and Hoster settings tabs. Only admins
|
||||
see administration; personal settings do not poll scraper services. See
|
||||
[settings navigation](docs/settings-navigation.md) for routes and verification.
|
||||
|
||||
### Settings (environment variables)
|
||||
|
||||
@@ -37,6 +64,10 @@ docker compose up -d --build
|
||||
| `SIGNUP` | `closed` | `open` lets anyone create an account. Otherwise the admin adds users under Settings |
|
||||
| `SCAN_INTERVAL` | `10` | Minutes between automatic library rescans (`0` = only at start and on "Rescan") |
|
||||
| `COOKIE_SECURE` | – | `1` marks the login cookie `Secure` (use it when you serve over HTTPS) |
|
||||
| `SCRAPER_URL` | `http://127.0.0.1:8091` | Private scraper daemon URL |
|
||||
| `SCRAPER_TOKEN` | empty | Shared daemon token; required when it listens beyond loopback |
|
||||
| `SCRAPER_TOKEN_FILE` | empty | Alternative secret file (used in Compose) |
|
||||
| `SENTINEL_URL` | `http://127.0.0.1:8092` | Private observation service URL |
|
||||
|
||||
Serve it behind a reverse proxy with HTTPS if it is reachable from the internet. All pages use relative URLs, so a sub-path like `/manga/` works.
|
||||
|
||||
@@ -63,12 +94,22 @@ library/
|
||||
|
||||
## Documentation
|
||||
|
||||
Unnumbered MangaDex pilots/one-shots use stable `special-mangadex-<chapter-uuid>`
|
||||
chapter keys and human labels, not fabricated numbers. Existing numeric IDs remain
|
||||
unchanged. In Settings → Admin → Scraper, **Inspect & queue** shows the source edition
|
||||
before **Confirm download**. “Job finished” refers to its selected scope, not verified
|
||||
whole-series completeness. See the scraper service and library format contracts below.
|
||||
|
||||
| Document | For |
|
||||
|----------|-----|
|
||||
| [docs/library-format.md](docs/library-format.md) | The folder format: naming rules, page tags, metadata fields, versions, extras, what survives a rename |
|
||||
| [docs/import.md](docs/import.md) | How the library is scanned and indexed, caching, change detection and notifications, troubleshooting |
|
||||
| [docs/architecture.md](docs/architecture.md) | Components, data model, API, security, the reader's layout engine, deployment, design decisions |
|
||||
| [docs/local-deployment.md](docs/local-deployment.md) | Current local Compose deployment, Tailscale access, persistent state, checks, restart, and recovery |
|
||||
| [docs/settings-navigation.md](docs/settings-navigation.md) | User/Admin settings sections, scraper tabs, routes and UI checks |
|
||||
| [docs/reading-insights-concept.md](docs/reading-insights-concept.md) | Active-time presentation, private summaries, deletion/export controls, and the future journal design |
|
||||
| [docs/scraper-guide.md](docs/scraper-guide.md) | Rule book (agent-readable) for scrapers and other tools that write into the library |
|
||||
| [scraper/AGENT_RUNBOOK.md](scraper/AGENT_RUNBOOK.md) | Conservative scraper workflow and host-specific procedures for automated agents |
|
||||
|
||||
---
|
||||
|
||||
@@ -81,7 +122,10 @@ library/
|
||||
| **Book** | Two facing pages, in reading order (right page first for `rtl`). Covers and double pages stand alone. Pairing restarts after each of them, so one odd page never shifts the whole chapter. If pairs still look wrong, **⇄ Shift** moves the pairs of the current section by one page (remembered per chapter) |
|
||||
|
||||
- **Which mode opens:** Settings → *Open chapters in* = a fixed mode or *the mode I used last*. With *Use the fitting mode…* on, content that isn't made for your choice opens in the mode that fits (webtoons always open in Flow). That automatic switch never changes your remembered mode. Only clicking a mode in the reader does.
|
||||
- **Page zoom:** 100 % means the page fits the screen (in Flow: a normal single page fits the screen; for webtoons, the strip is as wide as the screen but never wider than its own width). Pages may end up larger than the screen, exactly fitting, or smaller. Zoom with Ctrl + mouse wheel, touchpad pinch or two-finger pinch on the pages, or with `+` / `-` / `0`. This zooms only the pages; zooming over the toolbar zooms the whole page as usual. When a page is larger than the screen, drag it with the mouse (hand cursor) or pan with a finger. The zoom is remembered per mode for your account. Click the `100%` button to fit again.
|
||||
- **Page zoom:** 100 % means the page fits the screen (in Flow: a normal single page fits the screen; for webtoons, the strip is as wide as the screen but never wider than its own width). Pages may end up larger than the screen, exactly fitting, or smaller. The toolbar −/+ buttons and keyboard `-`/`+` move in 5-percentage-point steps from 20 % through 600 %; `0` or the percentage button returns to 100 %. Ctrl + mouse wheel, touchpad pinch and two-finger pinch remain continuous for still finer control. This zooms only the pages; zooming over the toolbar zooms the whole page as usual. When a page is larger than the screen, drag it with the mouse (hand cursor) or pan with a finger. Flow locks horizontal movement while the content fits within the viewport, then enables it automatically when zoom makes the content wider. The zoom is remembered per mode for your account.
|
||||
- **Flow page gap:** Settings → Reading lets each user keep the normal separator between Flow pages or remove it for continuous webtoon artwork and text that crosses image edges.
|
||||
- **Active Reading Clock:** opt in under Settings → User → Reading time & privacy. No scrolling-speed limit; a visit counts at 60% viewed-page coverage and saves on exit. Pauses in the background or after 30 seconds without reader activity. Tap the clock for this visit's time, coverage, qualification and loading wait.
|
||||
- **Your Reading Story:** private active-time summaries appear on the library and manga pages when history exists. Chapter rows show the last timed read (60%-coverage visit), rereads remain visible as lifetime time, and Settings lets each user hide, export, or delete their own history. No leaderboards or speed ranking are used.
|
||||
- **Tap / click zones:**
|
||||
- Tapping **beside the pages** always shows or hides the controls.
|
||||
- On a page, its left and right part turn the page, following the reading direction. For `rtl`, the left part goes forward.
|
||||
@@ -94,6 +138,8 @@ library/
|
||||
- **Loading:** the next pages (next 3 in Page mode, next 2 spreads in Book mode) are downloaded and decoded in advance, so page turns don't flash. A page that still isn't there shows a spinner. On phones, "Add to home screen" opens the app without browser bars, and the ⛶ button (where supported) goes fullscreen.
|
||||
- **Progress** is saved per user and chapter version: the last page, whether the chapter is done, and book-mode shifts. The manga page offers *Continue*. Tap a chapter's status to mark it read/unread.
|
||||
- **Notifications:** press *Notify me* on a manga. When a rescan finds new chapter numbers, they show up under the bell. Optionally, the server also POSTs a short text to a webhook URL of your choice (e.g. an ntfy.sh or Gotify topic, set in Settings).
|
||||
- **Finding manga:** the library can search titles, alternative titles, and authors; filter by publication status; combine tags with AND matching; and sort by title, activity, or first release date. Tag links on a manga open the library with that tag selected.
|
||||
- **Manga overview:** metadata is shown in a structured panel (with the complete raw `manga.json` available for inspection). Chapters are newest-first, with multi-select volume filters; selecting no volume means all volumes.
|
||||
|
||||
## Files
|
||||
|
||||
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
'use strict';
|
||||
|
||||
// Reader-owned catalog. Membership never changes filesystem or reading IDs.
|
||||
const fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||
const TYPES = ['prequel', 'sequel', 'side-story', 'spin-off', 'adaptation', 'alternate-version', 'alternate-timeline', 'compilation', 'remake', 'shared-universe', 'other'];
|
||||
const ROLES = ['main', ...TYPES];
|
||||
const STATES = ['suggested', 'confirmed', 'rejected', 'conflicted'];
|
||||
class CatalogError extends Error { constructor(code, message) { super(message); this.code = code; } }
|
||||
const fail = message => { throw new CatalogError(400, message); };
|
||||
const text = (value, max = 200) => String(value || '').trim().slice(0, max);
|
||||
function normalize(input, mangaIds, prior = {}) {
|
||||
if (!input || !Array.isArray(input.collections) || !Array.isArray(input.relations)) fail('Expected collections and relations arrays');
|
||||
if (input.collections.length > 2000 || input.relations.length > 10000) fail('Catalog is too large');
|
||||
const seen = new Set(), membership = new Set();
|
||||
const collections = input.collections.map(c => {
|
||||
const id = text(c.id || crypto.randomUUID(), 80);
|
||||
if (!/^[a-zA-Z0-9_-]+$/.test(id) || seen.has(id)) fail('Collection IDs must be unique letters, numbers, dash or underscore');
|
||||
seen.add(id);
|
||||
if (!text(c.title)) fail('Collection title is required');
|
||||
if (!Array.isArray(c.members) || !c.members.length) fail('A collection needs members');
|
||||
const local = new Set();
|
||||
const members = c.members.map((member, index) => {
|
||||
const mangaId = text(member.mangaId, 500);
|
||||
// Keep absent members on subsequent edits: an temporarily unavailable mount must not erase them.
|
||||
const wasMember = (prior.collections || []).some(old => old.id === id && old.members.some(m => m.mangaId === mangaId));
|
||||
if (!mangaIds.has(mangaId) && !wasMember) fail(`Unknown manga: ${mangaId}`);
|
||||
if (local.has(mangaId) || membership.has(mangaId)) fail(`Manga already belongs to a collection: ${mangaId}`);
|
||||
local.add(mangaId); membership.add(mangaId);
|
||||
const role = text(member.role || 'main');
|
||||
if (!ROLES.includes(role)) fail('Unknown member role');
|
||||
const readingOrder = Number(member.readingOrder ?? index + 1);
|
||||
if (!Number.isFinite(readingOrder) || readingOrder < 0) fail('Reading order must be a nonnegative number');
|
||||
return { mangaId, role, readingOrder };
|
||||
}).sort((a, b) => a.readingOrder - b.readingOrder);
|
||||
const coverMode = c.coverMode || 'selected';
|
||||
if (!['selected', 'mosaic', 'custom'].includes(coverMode)) fail('Unknown cover mode');
|
||||
const coverMangaId = text(c.coverMangaId || members[0].mangaId, 500);
|
||||
if (!local.has(coverMangaId)) fail('Selected cover must belong to this collection');
|
||||
const customCover = text(c.customCover, 100);
|
||||
if (customCover && !/^[a-f0-9]{64}\.(jpg|png|webp|gif|avif|bmp)$/.test(customCover)) fail('Invalid custom cover');
|
||||
return { id, title: text(c.title), description: text(c.description, 5000), members, coverMode, coverMangaId,
|
||||
...(customCover ? { customCover } : {}), grouped: c.grouped !== false };
|
||||
});
|
||||
const relationKeys = new Set();
|
||||
const relations = input.relations.map(r => {
|
||||
const from = text(r.from, 500), to = text(r.to, 500), type = text(r.type), state = r.state || 'suggested';
|
||||
if (from === to || !from || !to || !TYPES.includes(type) || !STATES.includes(state)) fail('Invalid relation');
|
||||
for (const id of [from, to]) if (!mangaIds.has(id) && !(prior.relations || []).some(old => old.from === id || old.to === id)) fail(`Unknown manga: ${id}`);
|
||||
const key = JSON.stringify([from, to, type]);
|
||||
if (relationKeys.has(key)) fail('Duplicate relationship');
|
||||
relationKeys.add(key);
|
||||
const evidence = (Array.isArray(r.evidence) ? r.evidence : []).slice(0, 12).map(e => {
|
||||
const url = text(e.url, 1000);
|
||||
if (url && !/^https?:\/\/[^\s]+$/i.test(url)) fail('Evidence must be an HTTP URL');
|
||||
return { provider: text(e.provider), url, note: text(e.note, 1500) };
|
||||
});
|
||||
return { from, to, type, state, evidence, explanation: text(r.explanation, 3000), method: text(r.method || 'manual'),
|
||||
confidence: Math.max(0, Math.min(1, Number(r.confidence) || 0)) };
|
||||
});
|
||||
return { version: 1, collections, relations };
|
||||
}
|
||||
|
||||
class Catalog {
|
||||
constructor(dataRoot) {
|
||||
this.file = path.join(dataRoot, 'catalog.json');
|
||||
this.state = { version: 1, revision: 0, collections: [], relations: [], audit: [] };
|
||||
if (fs.existsSync(this.file)) {
|
||||
const saved = JSON.parse(fs.readFileSync(this.file, 'utf8'));
|
||||
if (saved.version !== 1 || !Array.isArray(saved.collections) || !Array.isArray(saved.relations)) throw new Error('Unsupported catalog file');
|
||||
this.state = saved;
|
||||
}
|
||||
}
|
||||
update(input, mangaIds, actor) {
|
||||
if (input.revision !== this.state.revision) throw new CatalogError(409, 'Catalog changed; reload before saving');
|
||||
const value = normalize(input, mangaIds, this.state);
|
||||
const next = { ...value, revision: this.state.revision + 1,
|
||||
audit: [...(this.state.audit || []), { at: new Date().toISOString(), actor, revision: this.state.revision + 1 }].slice(-200) };
|
||||
fs.writeFileSync(this.file + '.tmp', JSON.stringify(next, null, 2) + '\n');
|
||||
fs.renameSync(this.file + '.tmp', this.file);
|
||||
this.state = next;
|
||||
return next;
|
||||
}
|
||||
suggest(mangas) {
|
||||
const byAniList = new Map(mangas.filter(m => m.meta?.['x-anilist']?.id).map(m => [String(m.meta['x-anilist'].id), m]));
|
||||
const relations = [...this.state.relations];
|
||||
const mapping = { PREQUEL: 'prequel', SEQUEL: 'sequel', SIDE_STORY: 'side-story', SPIN_OFF: 'spin-off', ADAPTATION: 'adaptation', ALTERNATIVE: 'alternate-version', COMPILATION: 'compilation', OTHER: 'other', SOURCE: 'adaptation', PARENT: 'other' };
|
||||
for (const m of mangas) for (const r of m.meta?.['x-anilist']?.relations || []) {
|
||||
const target = byAniList.get(String(r.id)), type = mapping[r.type] || 'other';
|
||||
if (!target || target.id === m.id || relations.some(e => e.from === m.id && e.to === target.id && e.type === type)) continue;
|
||||
relations.push({ from: m.id, to: target.id, type, state: 'suggested', method: 'anilist', confidence: 1,
|
||||
explanation: `AniList identifies the target as ${r.type}. Review before grouping.`,
|
||||
evidence: [{ provider: 'AniList', url: m.meta['x-anilist'].url || `https://anilist.co/manga/${m.meta['x-anilist'].id}`, note: r.type }] });
|
||||
}
|
||||
return relations;
|
||||
}
|
||||
}
|
||||
|
||||
function collectionViews(catalog, mangas) {
|
||||
const byId = new Map(mangas.map(m => [m.id, m]));
|
||||
return catalog.collections.map(c => {
|
||||
const members = c.members.filter(member => byId.has(member.mangaId)).map(member => ({ ...byId.get(member.mangaId), role: member.role, readingOrder: member.readingOrder }));
|
||||
const chosen = byId.get(c.coverMangaId) || members[0];
|
||||
const sum = key => members.reduce((n, m) => n + (+m[key] || 0), 0);
|
||||
const dates = members.map(m => m.firstReleaseAt).filter(Boolean);
|
||||
return { ...c, members, cover: c.coverMode === 'custom' && c.customCover ? `api/collection-cover?id=${encodeURIComponent(c.id)}&v=${c.customCover}` : chosen?.cover,
|
||||
covers: members.map(m => m.cover).filter(Boolean).slice(0, 4), chapters: sum('chapters'), read: sum('read'), unread: sum('unread'),
|
||||
tags: [...new Set(members.flatMap(m => m.tags || []))], authors: [...new Set(members.flatMap(m => m.authors || []))],
|
||||
altTitles: members.flatMap(m => [m.title, ...(m.altTitles || [])]),
|
||||
status: members.some(m => m.status === 'ongoing') ? 'ongoing' : members.every(m => m.status === 'completed') ? 'completed' : 'mixed',
|
||||
updated: Math.max(0, ...members.map(m => +m.updated || 0)), lastRead: Math.max(0, ...members.map(m => +m.lastRead || 0)),
|
||||
firstReleaseAt: dates.length ? Math.min(...dates) : 0, firstRelease: dates.length ? String(new Date(Math.min(...dates)).getUTCFullYear()) : '',
|
||||
reading: { activeMs: members.reduce((n, m) => n + (m.reading?.activeMs || 0), 0), updatedAt: Math.max(0, ...members.map(m => m.reading?.updatedAt || 0)) } };
|
||||
}).filter(c => c.members.length);
|
||||
}
|
||||
module.exports = { Catalog, CatalogError, collectionViews, normalize, TYPES, ROLES };
|
||||
@@ -0,0 +1,32 @@
|
||||
'use strict';
|
||||
|
||||
// An unnumbered release is not chapter zero. Keep its upstream identity intact.
|
||||
const UUID = '[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}';
|
||||
const SPECIAL = new RegExp(`^special-mangadex-(${UUID})$`, 'i');
|
||||
const SPECIAL_FOLDER = new RegExp(`^(special-mangadex-${UUID})(?: (.*))?$`, 'i');
|
||||
const NUMBERED_FOLDER = /^([a-z]*) ?(\d+(?:\.\d+)?)(?=$|[\s_\-[(])[\s_-]*(.*)$/i;
|
||||
const isSpecialKey = value => SPECIAL.test(String(value));
|
||||
function specialKey(id) {
|
||||
const key = `special-mangadex-${String(id).toLowerCase()}`;
|
||||
if (!isSpecialKey(key)) throw new Error('Unnumbered MangaDex release needs a stable chapter UUID');
|
||||
return key;
|
||||
}
|
||||
function parseSpecialFolder(name) {
|
||||
const match = SPECIAL_FOLDER.exec(name);
|
||||
if (!match) return null;
|
||||
let title = (match[2] || '').trim(), src = '';
|
||||
const source = /\s*\[([^\]]*)\]$/.exec(title);
|
||||
if (source) { src = source[1].trim(); title = title.slice(0, source.index).trim(); }
|
||||
return { key: match[1].toLowerCase(), special: true, x: false, num: null, title, src };
|
||||
}
|
||||
// Reader and scraper must assign the same identity to every numbered folder.
|
||||
function parseKey(name) {
|
||||
const match = NUMBERED_FOLDER.exec(name);
|
||||
if (!match) return null;
|
||||
const x = match[1].toLowerCase() === 'x', num = parseFloat(match[2]);
|
||||
let title = match[3].trim(), src = '';
|
||||
const source = /\s*\[([^\]]*)\]$/.exec(title);
|
||||
if (source) { src = source[1].trim(); title = title.slice(0, source.index).trim(); }
|
||||
return { key: (x ? 'x' : '') + num, x, num, title: title.replace(/^[-:.]\s*/, ''), src };
|
||||
}
|
||||
module.exports = { isSpecialKey, specialKey, parseSpecialFolder, parseKey };
|
||||
@@ -0,0 +1,15 @@
|
||||
# Controlled service reloads preserve checkpoints; public assets update immediately.
|
||||
services:
|
||||
reader:
|
||||
volumes:
|
||||
- ./public:/app/public:ro
|
||||
- ./server.js:/app/server.js:ro
|
||||
- ./catalog.js:/app/catalog.js:ro
|
||||
- ./reading-analytics.js:/app/reading-analytics.js:ro
|
||||
- ./library.js:/app/library.js:ro
|
||||
scraper:
|
||||
volumes:
|
||||
- ./scraper:/app/scraper:ro
|
||||
sentinel:
|
||||
volumes:
|
||||
- ./sentinel:/app/sentinel:ro
|
||||
@@ -0,0 +1,12 @@
|
||||
# Compose >= 2.24.4. Infrastructure owns the edge network.
|
||||
services:
|
||||
reader:
|
||||
ports: !reset []
|
||||
environment:
|
||||
COOKIE_SECURE: "1"
|
||||
networks: [backend, egress, edge]
|
||||
|
||||
networks:
|
||||
edge:
|
||||
external: true
|
||||
name: ${PROXY_NETWORK:-mangacircus-edge}
|
||||
+117
-14
@@ -1,15 +1,118 @@
|
||||
services:
|
||||
mangacircus:
|
||||
build: .
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./library:/library:ro # your manga folders (read-only is fine)
|
||||
- mangacircus-data:/data # users, progress, votes, scan cache
|
||||
environment:
|
||||
SIGNUP: closed # "open" lets anyone create an account
|
||||
SCAN_INTERVAL: 10 # minutes between automatic rescans (0 = off)
|
||||
restart: unless-stopped
|
||||
name: mangacircus
|
||||
|
||||
volumes:
|
||||
mangacircus-data:
|
||||
x-app: &app
|
||||
image: ${APP_IMAGE:-mangacircus:local}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-development}
|
||||
init: true
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 90s
|
||||
read_only: true
|
||||
tmpfs: [/tmp]
|
||||
security_opt: [no-new-privileges:true]
|
||||
cap_drop: [ALL]
|
||||
secrets: [scraper_token]
|
||||
logging:
|
||||
driver: local
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
services:
|
||||
reader:
|
||||
<<: *app
|
||||
command: [node, runtime/launch.js, reader]
|
||||
ports: ["${READER_BIND:-127.0.0.1}:${READER_PORT:-8080}:8080"]
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${LIBRARY_PATH:-./library}
|
||||
target: /library
|
||||
read_only: true
|
||||
bind: {create_host_path: false}
|
||||
- type: bind
|
||||
source: ${DATA_PATH:-./data}
|
||||
target: /data
|
||||
bind: {create_host_path: false}
|
||||
environment:
|
||||
PORT: "8080"
|
||||
DATA: /data
|
||||
LIBRARY: /library
|
||||
TZ: ${TZ:-Europe/Berlin}
|
||||
SIGNUP: closed
|
||||
SCAN_INTERVAL: "10"
|
||||
COOKIE_SECURE: ${COOKIE_SECURE:-0}
|
||||
SCRAPER_URL: http://scraper:8091
|
||||
SENTINEL_URL: http://sentinel:8092
|
||||
SCRAPER_TOKEN_FILE: /run/secrets/scraper_token
|
||||
networks: [backend, egress]
|
||||
healthcheck:
|
||||
test: [CMD, node, tools/healthcheck.js, reader]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
start_period: 60s
|
||||
retries: 3
|
||||
|
||||
scraper:
|
||||
<<: *app
|
||||
command: [node, runtime/launch.js, scraper]
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${LIBRARY_PATH:-./library}
|
||||
target: /library
|
||||
bind: {create_host_path: false}
|
||||
- type: bind
|
||||
source: ${SCRAPER_DATA_PATH:-./data/scraper}
|
||||
target: /data/scraper
|
||||
bind: {create_host_path: false}
|
||||
environment:
|
||||
DATA: /data
|
||||
LIBRARY: /library
|
||||
SCRAPER_DATA: /data/scraper
|
||||
METADATA_CACHE: /data/scraper/metadata/anilist
|
||||
TZ: ${TZ:-Europe/Berlin}
|
||||
SCRAPER_HOST: 0.0.0.0
|
||||
SCRAPER_PORT: "8091"
|
||||
SCRAPER_TOKEN_FILE: /run/secrets/scraper_token
|
||||
networks: [backend, egress]
|
||||
healthcheck:
|
||||
test: [CMD, node, tools/healthcheck.js, scraper]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
start_period: 60s
|
||||
retries: 3
|
||||
|
||||
sentinel:
|
||||
<<: *app
|
||||
command: [node, runtime/launch.js, sentinel]
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${SENTINEL_DATA_PATH:-./data/sentinel}
|
||||
target: /data/sentinel
|
||||
bind: {create_host_path: false}
|
||||
environment:
|
||||
DATA: /data
|
||||
SENTINEL_DATA: /data/sentinel
|
||||
SENTINEL_HOST: 0.0.0.0
|
||||
SENTINEL_PORT: "8092"
|
||||
READER_URL: http://reader:8080
|
||||
SCRAPER_URL: http://scraper:8091
|
||||
SCRAPER_TOKEN_FILE: /run/secrets/scraper_token
|
||||
TZ: ${TZ:-Europe/Berlin}
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: [CMD, node, tools/healthcheck.js, sentinel]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
start_period: 60s
|
||||
retries: 3
|
||||
|
||||
networks:
|
||||
backend:
|
||||
internal: true
|
||||
egress: {}
|
||||
|
||||
secrets:
|
||||
scraper_token:
|
||||
file: ${SCRAPER_TOKEN_PATH:-./.secrets/scraper_token}
|
||||
+81
-17
@@ -1,11 +1,20 @@
|
||||
# MangaCircus architecture
|
||||
|
||||
MangaCircus is a self-hosted reader for manga, comics and webtoons. One Node.js process serves a handful of static pages and a small JSON API. The content is a folder tree on disk, and everything user-related lives in one JSON file.
|
||||
MangaCircus is a self-hosted reader for manga, comics and webtoons. The reader owns
|
||||
accounts/progress and a separate collection catalog. The scraper owns the persistent
|
||||
queue and chapter publication. Sentinel observes health/progress and requests bounded
|
||||
checkpoint recovery. Separate supervisors isolate service failures. All content remains
|
||||
a plain folder tree; there is no database server or embedded autonomous LLM.
|
||||
|
||||
Related documents:
|
||||
- [library-format.md](library-format.md): the folder format.
|
||||
- [import.md](import.md): how the folder tree is indexed.
|
||||
- [scraper-guide.md](scraper-guide.md): rules for tools that write the library.
|
||||
- [scraper-service.md](scraper-service.md): managed queue, private API, and adapter contract.
|
||||
- [collections.md](collections.md): relations, groups, curator UI and agent research.
|
||||
- [resilience.md](resilience.md): supervision, recovery limits and incidents.
|
||||
- [compose-deployment.md](compose-deployment.md): local/VM deployment and release workflow.
|
||||
- [settings-navigation.md](settings-navigation.md): User/Admin workspace, scraper tabs and deep links.
|
||||
|
||||
## 1. Principles
|
||||
|
||||
@@ -13,7 +22,7 @@ Related documents:
|
||||
|-----------|-------------|
|
||||
| **The file system is the source of truth for content** | No import step, no copies, no content database. The library can be mounted read-only. Anything that writes files can feed it |
|
||||
| **As dumb as possible** | No dependencies (`package.json` has none), no framework, no build step, no transpiler. Plain ES2019+ JavaScript in the browser, plain Node on the server |
|
||||
| **One process, one data file** | In-memory library index, user data in `data/db.json`. Back up one file, run one container |
|
||||
| **Small independent services** | The reader stays dependency-free; the optional scraper daemon can stop or restart without taking reading offline |
|
||||
| **Readable without the app** | The folder format works in any picture viewer. The app adds progress, layout and convenience, not lock-in |
|
||||
|
||||
## 2. System context
|
||||
@@ -35,6 +44,18 @@ Related documents:
|
||||
|
||||
## 3. Components
|
||||
|
||||
Runtime topology (2026-10-05): native `tools/start-all.js` launches reader, scraper and
|
||||
Sentinel independently. In Compose, each container runs `runtime/launch.js <service>`
|
||||
and exactly one service child. A failure does not stop unrelated services. Exactly one
|
||||
reader and queue-owning scraper may write each shared state tree; do not scale replicas.
|
||||
The catalog and relation store is reader-owned `data/catalog.json`. Sentinel writes only
|
||||
`data/sentinel/`; it has no Docker socket or arbitrary code-repair authority.
|
||||
|
||||
The admin browser never talks to the scraper daemon directly. Admin-only reader
|
||||
routes proxy commands to the daemon's private API, so its shared token stays on the
|
||||
server. The daemon publishes complete folders atomically and increments a durable
|
||||
revision; the reader polls that revision and rescans the library after publication.
|
||||
|
||||
| Path | Lines | Responsibility |
|
||||
|------|------:|----------------|
|
||||
| `server.js` | ~350 | HTTP server and router, auth and sessions, JSON storage, API handlers, scan scheduling, notifications and webhooks |
|
||||
@@ -43,7 +64,7 @@ Related documents:
|
||||
| `public/index.html` + `library.js` | ~60 | Library overview (cover grid, filter, sort) |
|
||||
| `public/manga.html` + `manga.js` | ~110 | Manga page: info, follow, chapter list with book separators, versions and votes, read/unread toggle |
|
||||
| `public/read.html` + `reader.js` | ~470 | The reader (§7.2) |
|
||||
| `public/settings.html` + `settings.js` | ~115 | Reading preferences, webhook, password, admin (rescan, users) |
|
||||
| `public/settings.html` + `settings.js` + `settings-nav.js` | — | User/Admin navigation, personal preferences, lazy admin sections and scraper tabs |
|
||||
| `public/login.html` + `login.js` | ~50 | Sign-in, first-run admin creation, optional sign-up |
|
||||
| `public/app.css` | ~150 | All styles (dark theme, responsive) |
|
||||
| `tools/png.js`, `tools/make-dummy.js` | ~310 | Demo library generator: a tiny PNG encoder with a bitmap font, no image libraries |
|
||||
@@ -51,14 +72,15 @@ Related documents:
|
||||
|
||||
## 4. Runtime model
|
||||
|
||||
- **One process, event loop only.** All disk I/O in the scanner is asynchronous and sequential, so a scan never blocks requests.
|
||||
- **Reader process:** one event loop. All disk I/O in the scanner is asynchronous and sequential, so a scan never blocks requests.
|
||||
- **Optional scraper process:** one queue worker, one remote request at a time, durable checkpoints, and a private control API.
|
||||
- **In-memory state:**
|
||||
- `lib` is the current library index, replaced as a whole after each scan.
|
||||
- `db` is all user data, loaded from `db.json` at start.
|
||||
- **Timers:**
|
||||
- a library scan at start and every `SCAN_INTERVAL` minutes,
|
||||
- a debounced `db.json` write, 500 ms after the last change.
|
||||
- **Shutdown:** `SIGINT` / `SIGTERM` flush `db.json` synchronously, then exit. That matters for `docker stop`.
|
||||
- **Shutdown:** `SIGINT` / `SIGTERM` flush `db.json`; the daemon cooperatively stops at a page boundary and keeps its archive checkpoint.
|
||||
- **Durability:** every file write goes to `*.tmp` first and is then renamed over the target, so a file is never half-written. A hard kill loses at most the changes of the last ~0.5 s. A corrupt `db.json` makes the server **refuse to start** rather than start empty and overwrite it.
|
||||
|
||||
## 5. Data model
|
||||
@@ -93,6 +115,9 @@ This keeps ids human-readable and derivable without a database. The price: renam
|
||||
"lastMode": "book", // the user's intended mode; changed only by clicking a mode
|
||||
"adapt": true, // use a fitting mode for incompatible content (webtoon → flow)
|
||||
"skipExtras": false, "skipCovers": false,
|
||||
"flowPageGap": true, // show the 8 px separator between pages in Flow
|
||||
"readingClock": false, // opt in to credible per-chapter active-time tracking
|
||||
"readingStory": true, // show private retrospective summaries when history exists
|
||||
"zoom": { "flow": 1, "page": 1.5, "book": 1, "strip": 0.8 }, // page zoom per mode
|
||||
"webhook": ""
|
||||
}
|
||||
@@ -106,6 +131,12 @@ This keeps ids human-readable and derivable without a database. The price: renam
|
||||
"done": false, // sticky once true (reset via "mark unread")
|
||||
"shift": [] // book mode: run starts shown alone
|
||||
} } } },
|
||||
"reading": { "admin": { "Moon Garden": { "1/2": {
|
||||
"schema": 1, "totalActiveMs": 412000, "loadingMs": 7400,
|
||||
"sessions": 3, "completedPasses": 2,
|
||||
"inputMs": { "mouse": 12000, "touch": 380000, "keyboard": 20000 },
|
||||
"currentPass": { "version": "c0002 Roots [Official]", "seenRanges": [[0, 13]], "activeMs": 151000 }
|
||||
} } } },
|
||||
"follows": { "admin": { "Moon Garden": true } },
|
||||
"notes": { "admin": [ { "id": "a1b2c3d4e5f6", "t": 1790797300000, "m": "Moon Garden", "c": "2/6",
|
||||
"text": "Moon Garden · Ch. 6: Frost", "read": false } ] },
|
||||
@@ -142,10 +173,13 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js
|
||||
| `GET /api/me` | user | Name, admin flag, settings, unread count |
|
||||
| `POST /api/settings` `{…partial}` | user | Validated update of the settings keys above |
|
||||
| `POST /api/password` `{old, new}` | user | Change password |
|
||||
| `GET /api/library` | user | Manga list with covers, counts, the user's read count, unread notifications, last read |
|
||||
| `GET /api/manga?m=` | user | Manga details, books, chapters with ranked versions and the user's progress, `continue` target |
|
||||
| `GET /api/chapter?m=&c=[&v=]` | user | Pages (`u, w, h, t, r`), direction, versions, prev/next, progress, plus all books and chapters of the manga (with the user's read state) for the chapter picker. Also marks that chapter's notifications read |
|
||||
| `GET /api/library` | user | Manga list with covers, counts, the user's read count, unread notifications, last read, and optional personal Reading Story aggregates |
|
||||
| `GET /api/manga?m=` | user | Manga details, books, chapters with ranked versions, progress, optional chapter-time summaries, and `continue` target |
|
||||
| `GET /api/chapter?m=&c=[&v=]` | user | Pages (`u, w, h, t, r`), direction, versions, prev/next, progress, optional Active Reading Clock summary, plus all books and chapters of the manga (with the user's read state) for the chapter picker. Also marks that chapter's notifications read |
|
||||
| `POST /api/progress` `{m, c, v, p, n, done, shift}` | user | Save the reading position |
|
||||
| `POST /api/reading/pulse` `{m, c, v, session, seq, …deltas}` | user | When opted in, idempotently merge a clamped active-time pulse, verified page coverage and input totals under a single-reader lease |
|
||||
| `GET /api/reading/export` | user | Export the user's private derived reading-time summary without raw input or page-exposure diagnostics |
|
||||
| `POST /api/reading/delete` `{m}` or `{all:true,password}` | user | Delete one manga's reading history, or all history after password confirmation, without changing normal reading progress |
|
||||
| `POST /api/unread` `{m, c}` | user | Forget progress for a chapter |
|
||||
| `POST /api/vote` `{m, c, v, value: 1\|-1\|0}` | user | Vote a version; returns the re-ranked versions |
|
||||
| `POST /api/follow` `{m, on}` | user | Follow a manga for notifications |
|
||||
@@ -237,9 +271,9 @@ mode = adapt && intended ∉ compatible ? compatible[0] : intended
|
||||
| Click or tap on a page, top/bottom 35 % of the screen (flow) | Scroll to the previous/next page top, at most 90 % of a screen (85 % steps for webtoons). At the very bottom → next chapter |
|
||||
| Click or tap on a page, middle band | Toggle the bars |
|
||||
| Horizontal swipe ≥ 50 px within 800 ms (paged, not zoomed in) | Previous/next |
|
||||
| Ctrl + wheel, touchpad pinch, Safari gesture events, two-finger pinch | Page zoom 20–600 %, snapping to 100 % within ±3 %, anchored at the pointer or finger midpoint. Browser zoom is suppressed only over the pages |
|
||||
| Ctrl + wheel, touchpad pinch, Safari gesture events, two-finger pinch | Continuous page zoom 20–600 %, snapping to 100 % within ±1 %, anchored at the pointer or finger midpoint. Browser zoom is suppressed only over the pages |
|
||||
| Mouse drag (moved ≥ 6 px) | Pan; the following click is swallowed. Hand cursor when the content is larger than the screen |
|
||||
| Keys | Arrows follow the direction; Space/PageDown/PageUp (paged); `+` `-` `0` zoom; `m`/Esc bars; `f` fullscreen |
|
||||
| Keys | Arrows follow the direction; Space/PageDown/PageUp (paged); `+` and `-` zoom in 5-percentage-point steps, `0` fits at 100 %; `m`/Esc bars; `f` fullscreen |
|
||||
|
||||
**Bars:**
|
||||
- Shown on open.
|
||||
@@ -260,22 +294,50 @@ mode = adapt && intended ∉ compatible ? compatible[0] : intended
|
||||
- Stores the page index in the full page list, so skipping extras doesn't corrupt it.
|
||||
- On open, the saved page is used for the same version, and mapped proportionally for another version. A finished chapter reopens at the start.
|
||||
|
||||
**Active Reading Clock (opt-in):**
|
||||
- Runs only while the reader is visible and focused, a sufficiently exposed page is decoded, a server lease is held, and accepted reader input occurred in the last 50 seconds.
|
||||
- Backgrounding or blurring pauses immediately and requires fresh input to resume. A delayed callback can never add time beyond the 50-second input deadline.
|
||||
- A page is verified after at least 35% meaningful exposure for 200 ms cumulatively. Exposure is closed and evaluated when the page leaves the viewport, so a fast but plausible webtoon scroll is not lost between samples. A five-pages-per-second token bucket rejects implausible coverage without discarding otherwise credible active time.
|
||||
- `BroadcastChannel` coordinates tabs on one browser. The authenticated API adds a 15-second per-user lease for other devices, and releases it when the tab pauses or leaves.
|
||||
- Pulses are sent at most every 10 seconds and on pause/page exit. The server validates the manga/chapter/version and page indices, de-duplicates `(session, seq)`, clamps time to server-observed intervals and stores compact page ranges separately from ordinary progress.
|
||||
- The bottom toolbar chip shows active time and the current state; its panel adds pass time, verified pages, predominant input and loading wait.
|
||||
|
||||
**Zoom persistence:** per mode key (`flow`, `page`, `book`, `strip`), saved 800 ms after the last change.
|
||||
|
||||
## 8. Deployment
|
||||
|
||||
### Current Windows deployment
|
||||
|
||||
The observed local installation uses the Windows Scheduled Task
|
||||
`\MangaCircus Supervisor`. At logon it runs `tools/run-supervisor.ps1`, which appends
|
||||
logs under `data/scraper/` and starts one `tools/start-all.js` supervisor. That parent
|
||||
owns exactly one reader (`server.js`, port 8080) and one scraper daemon
|
||||
(`scraper/daemon.js`, private port 8091). Starting `npm run start:all` manually at the
|
||||
same time would create conflicting writers and is not supported.
|
||||
|
||||
The scraper's authoritative `data/scraper/queue.json` is schema version 3. It supports
|
||||
legacy archive checkpoints and paused conversion of completed legacy work while adding
|
||||
source-aware lanes, scoped queue jobs, incremental chapter publication, and nightly catalog discovery. See
|
||||
[local-deployment.md](local-deployment.md) for operational checks and restart/recovery,
|
||||
and [scraper-service.md](scraper-service.md) for the queue/API contract.
|
||||
|
||||
### Docker or manual deployment
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
- **Image:** `node:24-alpine` plus the app files. It runs as user `node` (uid 1000) and has a health check against `/api/auth`.
|
||||
- **Images/services:** the same `node:24-alpine` image runs a reader service and a scraper service. The reader health check targets `/api/auth`.
|
||||
- **Volumes:**
|
||||
- `/library`: read-only is enough. Files must be readable by uid 1000.
|
||||
- `/data`: must be writable.
|
||||
- **Configuration:** only environment variables (`PORT`, `LIBRARY`, `DATA`, `SIGNUP`, `SCAN_INTERVAL`, `COOKIE_SECURE`).
|
||||
- **Without Docker:** `node server.js` with Node ≥ 18.
|
||||
- `/library`: read-only in the reader and writable in the scraper. Files must be readable—and scraper targets writable—by uid 1000.
|
||||
- `/data`: reader state must be writable; the scraper also needs its writable `/data/scraper` state.
|
||||
- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `SIGNUP`, `SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI.
|
||||
- **Without Docker:** `npm run start:all` with Node ≥ 18, or supervise `npm start` and `npm run start:scraper` independently. On the current Windows machine that supervision is already owned by `\MangaCircus Supervisor`.
|
||||
- **Behind a reverse proxy:** terminate TLS, set `COOKIE_SECURE=1`. A sub-path works.
|
||||
- **Upgrades:** replace the code and restart. There are no migrations: unknown fields in `db.json` are kept, and missing settings fall back to defaults.
|
||||
- **Upgrades:** replace the code and restart. Reader state has no explicit migration;
|
||||
unknown fields in `db.json` are kept. The scraper normalizes old queue entries into
|
||||
schema version 3 on startup and preserves legacy job checkpoint behavior. Take a
|
||||
queue/checkpoint recovery copy before upgrading a live scraper.
|
||||
|
||||
## 9. Capacity and limits
|
||||
|
||||
@@ -308,5 +370,7 @@ docker compose up -d --build
|
||||
- **Notifications:** in-app plus webhook only. Web Push would need VAPID keys and a service worker.
|
||||
- **No login rate limiting and no CSP header.** Both are small additions in `server.js`.
|
||||
- **In-place image overwrites** are only detected by a full rescan.
|
||||
- **No automated test suite.** Verification so far was done with the demo library (`tools/make-dummy.js`), API smoke scripts and manual browser testing on desktop and mobile viewports. `parseKey` and `sizeFromBuffer` are pure functions, and `scanLibrary` only needs a folder. Together with the book pairing they are the natural first targets for unit tests.
|
||||
- **Reader layout remains mostly smoke/manual.** Reading analytics has focused Node tests
|
||||
for its persisted merge rules, while DOM layout, authentication and mobile browser
|
||||
lifecycle behavior still need browser-level/manual verification.
|
||||
- **Adding content by API** (instead of files) would be a new route plus the staging and renaming rules from [import.md §7](import.md#7-consistency-writing-into-a-live-library).
|
||||
@@ -0,0 +1,141 @@
|
||||
# Collections and relationships
|
||||
|
||||
Collections are a presentation layer, not a filesystem migration. Creating, reordering,
|
||||
or removing one never renames manga/chapter folders or changes reading IDs.
|
||||
|
||||
## Using the UI
|
||||
|
||||
Open **Settings → Admin → Collections & metadata** (or **Manage collections** on the dashboard).
|
||||
Create a collection, add library manga, choose each member's role and reading order,
|
||||
and save. Choose a member cover, a mosaic of up to four covers, or upload a custom
|
||||
image under 600 KB. Dashboard display can be grouped, expanded, or individual; the
|
||||
choice is remembered per browser. A collection page shows aggregate progress and
|
||||
continues the first unfinished member in the chosen order.
|
||||
|
||||
**New collection** starts an unsaved draft using an ungrouped library manga. Rename
|
||||
it, add members and choose **Save collection** (or **Save all changes**). Each manga
|
||||
can belong to one collection. If all manga are grouped, remove a member from another
|
||||
collection first; a collection's final member is retained until you delete the
|
||||
collection. Deletion never removes manga or reading progress.
|
||||
|
||||
Save feedback and errors appear above the editor. Controls are locked during a save;
|
||||
failed saves retain drafts. If another admin has changed the catalog, the stale write
|
||||
is rejected. **Reload catalog** fetches the latest revision, with confirmation before
|
||||
discarding edits; copy any wanted changes first. Unsaved JSON survives collection
|
||||
selection and must use **Validate & save JSON**, not an ordinary collection save.
|
||||
Metadata imports/research are blocked while drafts exist so they cannot silently
|
||||
replace them. Relationship edits save immediately when there is no collection draft.
|
||||
|
||||
Creation works on HTTPS, localhost and ordinary HTTP LAN/Tailscale addresses. The
|
||||
browser uses secure random bytes when its secure-context-only `randomUUID` API is
|
||||
unavailable. This does not change networking or the server's revision/validation rules.
|
||||
|
||||
Tag combinations match a single member, not tags scattered across unrelated members.
|
||||
Reading-time totals only include the current account's visible Reading Story. Collection
|
||||
publication status is ongoing if any member is ongoing, completed if all are completed,
|
||||
otherwise mixed. Collections do not control scraping or merge editions.
|
||||
|
||||
## Three distinct concepts
|
||||
|
||||
| Concept | Meaning | Owner |
|
||||
|---|---|---|
|
||||
| Relationship | Directed fact: target is a prequel/sequel/adaptation/etc. of source | Reader's Catalog Curator |
|
||||
| Collection | Which library manga share a dashboard tile | Admin decision |
|
||||
| Reading order | Recommended order within that collection | Admin decision, independent of chronology |
|
||||
|
||||
One manga belongs to at most one collection, preventing duplicate dashboard tiles and
|
||||
double-counted progress. Relationships can connect any number of works across groups.
|
||||
Only confirmed relationships appear on reader manga pages. Suggestions, rejections
|
||||
and conflicts remain visible to the admin. Removing a collection leaves its manga,
|
||||
reading history and relationships intact.
|
||||
|
||||
## Metadata and agent responsibilities
|
||||
|
||||
The metadata provider, **not the image scraper**, supplies relationships. AniList
|
||||
metadata now retains manga relationship edges in `manga.json` → `x-anilist.relations`.
|
||||
Existing metadata will not magically gain these fields: use **Research relationships**
|
||||
for a manga with a verified AniList ID, or import already-cached suggestions.
|
||||
Research uses one public metadata lookup with a 30-day cache, never image-host traffic.
|
||||
Only relationships whose two exact AniList identities already exist in the library
|
||||
become suggestions. This does not auto-enqueue missing manga or guess title matches.
|
||||
|
||||
An agent is a **catalog researcher/curator assistant**. It can read the admin JSON export,
|
||||
research public evidence, identify conflicting claims and propose relationships or
|
||||
membership edits. It should preserve rejected decisions and report uncertainty.
|
||||
It must not turn a shared-universe link into an automatic collection or an unquestioned
|
||||
reading order. No embedded LLM, API credentials, automatic research loop, or autonomous
|
||||
catalog approval is enabled in this release. That optional capability belongs behind
|
||||
this same evidence/review boundary, not inside adapters or Sentinel.
|
||||
|
||||
## Persistence and API contract
|
||||
|
||||
`data/catalog.json` is reader-owned, version 1, atomically replaced on save. Its
|
||||
revision supports optimistic concurrency; stale writes return 409. Unknown members,
|
||||
duplicate membership, invalid relationships and cover paths are rejected. Existing
|
||||
members absent from a temporary mount are retained rather than erased.
|
||||
Custom covers are content-addressed in `data/collection-covers/`. Include both in backups.
|
||||
|
||||
Authenticated reader APIs:
|
||||
|
||||
- `GET /api/collections`, `GET /api/collection?id=…` — account-specific public views.
|
||||
- `GET /api/admin/catalog` — admin export plus available manga, roles and relation types.
|
||||
- `POST /api/admin/catalog` — validated full replacement, with current `revision`.
|
||||
- `POST /api/admin/catalog/suggest` — import cached edges, with current revision.
|
||||
- `POST /api/admin/catalog/research` — `{m: libraryMangaId, revision}`; verified metadata lookup.
|
||||
- `POST /api/admin/collection-cover` — `{id, revision, base64}`; validated image upload.
|
||||
|
||||
Example proposal (use real IDs and the latest revision from the export):
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"revision": 0,
|
||||
"collections": [{
|
||||
"id": "moon-garden",
|
||||
"title": "Moon Garden Universe",
|
||||
"description": "Main story, then the prequel.",
|
||||
"grouped": true,
|
||||
"coverMode": "mosaic",
|
||||
"coverMangaId": "Moon Garden",
|
||||
"members": [
|
||||
{"mangaId": "Moon Garden", "role": "main", "readingOrder": 1},
|
||||
{"mangaId": "Before the Rain", "role": "prequel", "readingOrder": 2}
|
||||
]
|
||||
}],
|
||||
"relations": [{
|
||||
"from": "Moon Garden", "to": "Before the Rain", "type": "prequel",
|
||||
"state": "suggested", "method": "agent-research", "confidence": 0.8,
|
||||
"explanation": "Verify the publisher's chronology before accepting.",
|
||||
"evidence": [{"provider": "Publisher", "url": "https://example.com/work", "note": "Chronology"}]
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
Provider contract: [AniList relationship connections](https://docs.anilist.co/guide/graphql/connections).
|
||||
|
||||
## Repeatable functional acceptance
|
||||
|
||||
Run `npm.cmd test`, build `docker compose build reader`, then:
|
||||
|
||||
```powershell
|
||||
# Playwright is optional developer tooling; not a deployed application dependency.
|
||||
# Set NODE_PATH to its installed package root if it is not on normal resolution paths.
|
||||
node tools/test-compose.js --collections-ui --settings-ui
|
||||
```
|
||||
|
||||
The runner creates an isolated Compose project on loopback 8180 with three synthetic
|
||||
manga, disposable accounts, paused scraper and disabled discovery. It never mounts
|
||||
the real library, DB or queue. Headless Edge (override `QA_BROWSER_CHANNEL` if needed)
|
||||
proxies a synthetic `http://manga-qa.test:8180` origin to that fixture, reproducing the
|
||||
browser restrictions of a non-localhost HTTP address without using a real tailnet
|
||||
account. Screenshots: `data/qa/collections/desktop-editor.png` and `mobile-editor.png`.
|
||||
|
||||
Coverage: creation, validation, persistence after reload, member add/remove/role/order,
|
||||
selected/mosaic/custom covers and upload limits, JSON round-trip/draft protection,
|
||||
relationship creation/rejection/duplicates/removal, cached import, missing verified
|
||||
metadata identity, desktop/mobile edits, dashboard grouping and collection reading
|
||||
order, delete/cancel, overlapping saves, transient save failure/retry, concurrent-admin
|
||||
revision conflicts, and ordinary-reader read/write denial. No external manga or
|
||||
metadata research is performed; provider lookup success remains a separate test.
|
||||
The earlier settings test checked collections navigation only; this suite exercises
|
||||
the actual collection workflow and the insecure-origin failure it missed.
|
||||
+2
-2
@@ -91,7 +91,7 @@ Each chapter folder is one **version**:
|
||||
| `id` | folder name |
|
||||
| `source`, `sourceUrl` | `chapter.json.source` (`{name,url}` or a string), else the `[…]` part of the folder name |
|
||||
| `added` | `Date.parse(chapter.json.added)`, else the folder mtime |
|
||||
| `direction` | first valid of chapter.json → book.json → manga.json → `rtl`. Aliases: `vertical`, `webtoon` → `ttb` |
|
||||
| `direction` | manual chapter → book → manga; else coherent tall-strip dimensions → `ttb`; else prior generated vertical series direction; else fallback `rtl`. Aliases: `vertical`, `webtoon` → `ttb`. Legacy plain values remain manual |
|
||||
|
||||
6. A version without images is dropped silently.
|
||||
|
||||
@@ -196,7 +196,7 @@ The index is read by the API at request time. Nothing is precomputed per user.
|
||||
|
||||
| Endpoint | Built from |
|
||||
|----------|------------|
|
||||
| `GET /api/library` | manga list + the user's progress, follows and unread notifications |
|
||||
| `GET /api/library` | manga list + searchable metadata (`altTitles`, `authors`, `tags`, `status`, normalized first release), the user's progress, follows and unread notifications |
|
||||
| `GET /api/manga?m=` | manga incl. `meta` (the raw `manga.json`), books, chapters with ranked versions and the user's progress, "Continue" target |
|
||||
| `GET /api/chapter?m=&c=&v=` | pages of the chosen version (`v`, else the best ranked), direction, prev/next chapter (skipping extras if the user wants that), the user's progress |
|
||||
| `GET /lib/<path>` | the image file itself. Logged-in users only, image extensions only, path must stay inside `LIBRARY`. ETag + `Cache-Control: private, max-age=86400` |
|
||||
|
||||
+28
-1
@@ -72,6 +72,16 @@ Ignored, with a warning in the log:
|
||||
|
||||
### The key
|
||||
|
||||
The additive unnumbered MangaDex chapter form is
|
||||
`special-mangadex-<chapter-uuid> Pilot [MangaDex · Group]`. It is accepted only for
|
||||
chapters, never books. Its logical key is the complete lower-case prefix/UUID;
|
||||
`chapter.json` supplies `special:true`, `label` (Pilot/One-shot/Special), `sourceId`
|
||||
and nullable `sourceNumber`. The reader shows that label with no fabricated number.
|
||||
Such chapters sort after numbered entries and are not inherently skippable extras.
|
||||
Different source UUIDs remain distinct releases; no content equivalence is guessed.
|
||||
Numeric IDs and source-version folders are unchanged. Older reader releases ignore
|
||||
these new folders but retain them on disk; upgrade again to restore visibility.
|
||||
|
||||
The number is normalised: leading zeros and trailing decimal zeros are dropped (`0010.50` → `10.5`), and an `x` prefix is kept. The result is the **key**:
|
||||
|
||||
| Folder | Key |
|
||||
@@ -165,10 +175,17 @@ UTF-8 JSON (a BOM is tolerated). All fields are optional, and unknown fields are
|
||||
| `tags` | string[] | Genres / tags |
|
||||
| `description` | string | Synopsis (line breaks are kept) |
|
||||
| `direction` | `"rtl"` \| `"ltr"` \| `"ttb"` | How the pages are read. `rtl` (the default): manga, right to left. `ltr`: comics, printed manhwa. `ttb`: webtoon, one vertical strip cut into slices (aliases `vertical`, `webtoon`) |
|
||||
| `directionOverride` | same values | Explicit manual choice. Use this even when choosing `rtl`, which is also the fallback. May be set at manga, book, or chapter level |
|
||||
| `x-reading-direction` | object | Generated provenance: `{ "version": 1, "value": "ttb", "source": "detected" }` or `source: "fallback"`, with optional numeric evidence. The scraper owns this field; it is not a confidence score |
|
||||
| `sources` | `{name, url}[]` | Upstream sources the manga was scraped from. Several are allowed |
|
||||
| `x-anilist` | object | AniList work ID and linked metadata. The ID identifies a work, not necessarily a particular pilot or edition |
|
||||
| `x-metadata.imported` | object | Last imported values for title, alternative titles, authors, status, year, tags and description. The scraper updates a displayed field only when it is empty or still equals its previous imported value; edits in legacy files without this snapshot are preserved |
|
||||
| `x-scraper.autoUpdate` | boolean | Optional explicit admin choice. When absent, `status: "ongoing"` enables nightly checks and other statuses disable them |
|
||||
|
||||
The whole file is sent to logged-in users as part of the manga page. **Don't put secrets in it** (scraper tokens, cookies).
|
||||
|
||||
An AniList refresh with a different ID is stopped for review before chapter publication. Existing files that already contain `x-scraper.autoUpdate` keep that choice; the scraper does not guess whether an older value was automatic or manual. New files leave the choice absent so the default can follow status changes.
|
||||
|
||||
### `book.json`
|
||||
|
||||
| Field | Type | Meaning |
|
||||
@@ -188,7 +205,17 @@ The whole file is sent to logged-in users as part of the manga page. **Don't put
|
||||
| `priority` | number | Tie-breaker between versions with equal votes. Higher wins (default `0`) |
|
||||
| `direction` | as above | Overrides book/manga direction, e.g. a colour webtoon special inside a printed series |
|
||||
|
||||
`direction` is inherited **chapter → book → manga → `rtl`**, and each version can carry its own.
|
||||
Manual directions have priority at **chapter → book → manga**. A `directionOverride`
|
||||
or a plain `direction` without matching generated provenance is manual. Otherwise,
|
||||
the scanner checks the version's existing image dimensions for coherent tall strips
|
||||
and uses `ttb` when they are present. A prior generated vertical series direction
|
||||
applies to chapters with insufficient shape evidence. The final fallback is `rtl`;
|
||||
this does **not** classify a work as Japanese or prove its reading order. A direct
|
||||
edit to a generated `direction` becomes manual when it differs from its provenance.
|
||||
Existing plain directions are treated as manual because old scraper defaults and
|
||||
human edits cannot be distinguished safely. Set `directionOverride` to make a
|
||||
lasting choice, or explicitly review and migrate a legacy entry. Image files and
|
||||
chapter IDs do not change when direction metadata changes.
|
||||
|
||||
## Picture-viewer compatibility
|
||||
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
# Reading Story concept
|
||||
|
||||
Status: first release implemented on 2026-10-04. The Active Reading Clock records the
|
||||
underlying per-user, per-chapter measurements and MangaCircus now returns the compact
|
||||
library, manga and chapter summaries described below. The bounded daily Reading Journal
|
||||
remains a later phase.
|
||||
|
||||
Policy update 2026-10-06: a timed read now means a visit with at least 60% viewed
|
||||
page coverage, not exhaustive 100% verification. No speed ceiling applies. Below-
|
||||
threshold lookup visits do not enter these summaries; existing history is retained.
|
||||
First/last durations continue growing until departure. See `reading-time-concept.md`.
|
||||
|
||||
## Product idea
|
||||
|
||||
Call the retrospective feature **Your Reading Story**. The Active Reading Clock is the
|
||||
instrument visible while reading; Reading Story is the quiet, personal record it builds.
|
||||
|
||||
The feature should answer four pleasant questions:
|
||||
|
||||
1. How much time have I spent with this manga?
|
||||
2. What did this chapter take me to read?
|
||||
3. Where am I in the story, including partially timed chapters?
|
||||
4. What does my reading habit look like over time?
|
||||
|
||||
It is not a productivity score. Do not rank users, award points for longer sessions, or
|
||||
describe a reader as fast or slow. Those patterns would reward timer inflation and turn a
|
||||
private leisure activity into a competition.
|
||||
|
||||
## Display language
|
||||
|
||||
Use **active time**, **reading time**, and **timed**. Never call it screen time or claim
|
||||
that MangaCircus proved comprehension.
|
||||
|
||||
Durations are friendly rather than stopwatch-precise outside the live reader:
|
||||
|
||||
| Stored measurement | Display |
|
||||
|---|---|
|
||||
| no reading record | `Not timed` |
|
||||
| 1–59 seconds | `Under a minute` |
|
||||
| 60 seconds–59 minutes | rounded to the nearest minute, for example `7 min` |
|
||||
| 1 hour or more | hours and minutes, for example `3 h 42 min` |
|
||||
| incomplete current pass | `4 min active · in progress` |
|
||||
| one completed pass | `6 min first read` |
|
||||
| completed rereads | `5 min last read · 18 min total · 3 passes` |
|
||||
|
||||
Zero must not mean “read instantly.” A chapter with no record is **Not timed**, because
|
||||
the setting may have been disabled or the chapter may have been marked read manually.
|
||||
Loading wait and unqualified visit drafts remain diagnostic details; they are not mixed
|
||||
into the headline reading duration.
|
||||
|
||||
## 1. Reader: the live clock
|
||||
|
||||
Keep the implemented bottom-toolbar clock as the immediate view:
|
||||
|
||||
```text
|
||||
[ ⏱ 06:14 · Reading ]
|
||||
```
|
||||
|
||||
Tapping it opens the detail panel. At 60% coverage show a gentle confirmation:
|
||||
**Reading time is now counted · 60% of pages viewed**. Time continues accumulating;
|
||||
this does not claim the entire chapter has been read or alter ordinary progress.
|
||||
|
||||
For subsequent qualifying visits say **Reread time is now counted**.
|
||||
“Together” makes the lifetime number feel connected to the story rather than to work.
|
||||
|
||||
## 2. Manga page: Your Reading Story card
|
||||
|
||||
Place one card after the hero and before metadata. Hide it entirely when the user has no
|
||||
timing records for this manga; do not show a large empty dashboard.
|
||||
|
||||
```text
|
||||
┌ Your Reading Story ─────────────────────────────────────────┐
|
||||
│ 3 h 42 min together 31 chapters timed 4 rereads │
|
||||
│ Typical chapter 7 min Last read Ch. 45 · 18 min ago │
|
||||
│ │
|
||||
│ Reading rhythm │
|
||||
│ 01 ▂▂▃▂▅▃▂ 02 ▃▄▂▂▃ 03 ▂▂▆▃▂ │
|
||||
│ Each bar is one completed chapter; height is active time. │
|
||||
└────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
The headline values are:
|
||||
|
||||
- **time together:** sum of `totalActiveMs` for every chapter in this manga;
|
||||
- **chapters timed:** chapters with any credible active time;
|
||||
- **completed passes:** sum of completed passes, phrased as rereads only after subtracting
|
||||
the first completed pass per chapter;
|
||||
- **typical chapter:** median `lastCompleteMs` across completed chapters, never the mean;
|
||||
- **last read:** chapter with the newest `updatedAt`.
|
||||
|
||||
The **Reading rhythm** is a compact chapter sequence, grouped by the existing volume
|
||||
boundaries. It visualises only completed-pass duration. A gap means “not timed,” not zero.
|
||||
Bars use a log-like capped scale so one very long chapter does not flatten the rest.
|
||||
Hovering or tapping a bar reveals chapter number, title, last completed duration, lifetime
|
||||
time and pass count. Selecting a volume with the existing filter also filters this rhythm.
|
||||
|
||||
On narrow screens, show the first four headline values in a two-column grid and make the
|
||||
rhythm horizontally scrollable. The card is collapsible and remembers its state per user.
|
||||
|
||||
## 3. Chapter list: useful, quiet timing
|
||||
|
||||
Add the duration between the page count and chapter title, using a fixed-width column so
|
||||
the current chapter-row alignment stays clean:
|
||||
|
||||
```text
|
||||
Ch. 45 35 p. 7 min The Twin Swords ✓ read
|
||||
Ch. 44 31 p. 4 min The Visitor ✓ read
|
||||
Ch. 43 29 p. Not timed A Quiet Morning ✓ read
|
||||
```
|
||||
|
||||
The compact column shows the last completed pass. An incomplete chapter shows its current
|
||||
active time with a small hollow clock, such as `◷ 3 min`. A chapter with multiple passes
|
||||
gets a subtle repeat marker: `7 min ↻2`. Its accessible label and tooltip expand this to
|
||||
`Last read 7 minutes; 2 completed passes; 19 minutes total`.
|
||||
|
||||
Do not put lifetime time directly into every row; that makes reread chapters visually
|
||||
noisy. Lifetime detail belongs in the tooltip/tap sheet and Reading Story card.
|
||||
|
||||
The chapter timing column is hidden when the clock setting is disabled. Existing history
|
||||
is preserved and becomes visible again if the user re-enables the setting.
|
||||
|
||||
## 4. Library dashboard: one personal summary strip
|
||||
|
||||
Above the current filters and cover grid, show a compact strip only when at least one
|
||||
chapter has been timed:
|
||||
|
||||
```text
|
||||
Your Reading Story 18 h 26 min together · 146 chapters timed
|
||||
Most time with Berserk · Continue your last chapter →
|
||||
```
|
||||
|
||||
Add two optional library sorts:
|
||||
|
||||
- **Recently timed** — newest reading `updatedAt` first;
|
||||
- **Time together** — manga with the largest personal active-time total first.
|
||||
|
||||
Do not put duration badges on every cover by default. They compete with new-chapter badges
|
||||
and reading progress. A future `Show reading time on covers` preference may opt into a
|
||||
small `3 h 42 min` line below the existing read count.
|
||||
|
||||
The first release should not claim “today,” “this week,” streaks, or a calendar. The
|
||||
current compact database has totals and latest timestamps, but no honest time-series
|
||||
history from which those values can be reconstructed.
|
||||
|
||||
## 5. Later: Reading Journal
|
||||
|
||||
If readers enjoy Reading Story, add bounded daily aggregates rather than raw activity
|
||||
events:
|
||||
|
||||
```jsonc
|
||||
"daily": {
|
||||
"2026-10-04": { "activeMs": 2840000, "chapters": 6, "completedPasses": 5 }
|
||||
}
|
||||
```
|
||||
|
||||
Keep at most 400 daily buckets per user, folding older values into yearly totals. This
|
||||
enables a private activity calendar, `this week`, `this year`, and “you returned to this
|
||||
story after 3 months” without storing pointer movements, individual page timestamps or a
|
||||
surveillance-style event log.
|
||||
|
||||
Recommended journal messages are descriptive, not judgemental:
|
||||
|
||||
- **A long evening with Berserk · 1 h 18 min**
|
||||
- **You returned to Gantz after 42 days**
|
||||
- **Volume 4 was your longest volume · 2 h 06 min**
|
||||
- **12 chapters reread this year**
|
||||
|
||||
Avoid streak pressure, “goal missed,” speed comparisons, and global leaderboards.
|
||||
|
||||
## API and derived-data design
|
||||
|
||||
Keep the raw `reading` records private to their owner. Add derived summaries at the
|
||||
existing authenticated boundaries:
|
||||
|
||||
### `GET /api/library`
|
||||
|
||||
Each manga may include:
|
||||
|
||||
```jsonc
|
||||
"reading": {
|
||||
"activeMs": 13320000,
|
||||
"timedChapters": 31,
|
||||
"completedChapters": 29,
|
||||
"completedPasses": 35,
|
||||
"rereads": 6,
|
||||
"typicalChapterMs": 421000,
|
||||
"lastChapterId": "3/45",
|
||||
"updatedAt": 1791130200000
|
||||
}
|
||||
```
|
||||
|
||||
Also return a top-level personal total derived in the same pass. This avoids an extra
|
||||
request and lets library sorting work locally like the existing filters.
|
||||
|
||||
### `GET /api/manga`
|
||||
|
||||
Return the manga aggregate plus a small public summary on each chapter:
|
||||
|
||||
```jsonc
|
||||
"reading": {
|
||||
"totalActiveMs": 1140000,
|
||||
"lastCompleteMs": 421000,
|
||||
"completedPasses": 2,
|
||||
"currentPassMs": 0,
|
||||
"currentPassComplete": true,
|
||||
"updatedAt": 1791130200000
|
||||
}
|
||||
```
|
||||
|
||||
Do not send input-mode totals, rejected views or page ranges to the manga page. They are
|
||||
reader diagnostics and would make the overview payload and interface unnecessarily noisy.
|
||||
|
||||
Derive all aggregates on the server from the compact records during these requests. The
|
||||
library is small and JSON-backed; a second denormalised cache would create invalidation
|
||||
work without a current need. If measurement proves this slow, cache by a monotonic
|
||||
reading-data revision rather than writing duplicate totals into `db.json`.
|
||||
|
||||
## Controls, privacy and corrections
|
||||
|
||||
Under **Settings → Reading**, explain: “Reading Story is private to your account.” Add:
|
||||
|
||||
- **Show Reading Story** — controls retrospective UI separately from collecting the clock;
|
||||
- **Forget one manga’s reading time** — available on that manga's card menu;
|
||||
- **Delete all reading-time history** — destructive, confirmed with the account password;
|
||||
- **Export my reading-time summary** — compact JSON, with no raw input events.
|
||||
|
||||
Turning off the Active Reading Clock stops collection but does not silently erase history.
|
||||
The retrospective display setting may hide existing history without changing it. An admin
|
||||
must not see another user’s Reading Story through the normal admin interface.
|
||||
|
||||
## Delivery status
|
||||
|
||||
Implemented:
|
||||
|
||||
1. Server-side summary helpers and tests for missing, partial, complete and reread records.
|
||||
2. Manga-page card, volume-aware rhythm and aligned chapter-duration column.
|
||||
3. Library summary strip and the two personal-time sorts.
|
||||
4. User controls for hide, per-manga deletion, password-confirmed complete deletion and export.
|
||||
5. A gentle reader completion message for first reads and rereads.
|
||||
|
||||
Deferred: observe real use before adding bounded daily aggregates and the Reading Journal.
|
||||
|
||||
This order immediately makes the existing measurements useful without changing the clock
|
||||
protocol or committing MangaCircus to a larger analytics history.
|
||||
@@ -0,0 +1,84 @@
|
||||
# Active Reading Clock
|
||||
|
||||
Current policy: relaxed personal estimates, implemented 2026-10-06. This is not an
|
||||
anti-cheat system, proof of comprehension, or a basis for competitive rewards.
|
||||
Enable it in User → Reading time. Disabled means no clock requests.
|
||||
|
||||
## What counts
|
||||
|
||||
Each chapter visit measures foreground, loaded-page viewing time. After viewing
|
||||
**at least 60% of eligible pages**, rounded up, the whole visit's active time counts,
|
||||
including time before reaching that threshold. It continues counting through the
|
||||
rest of the visit and is saved periodically and when leaving the reader.
|
||||
|
||||
- There is **no pages-per-second ceiling** and no skimming penalty.
|
||||
- A loaded image needs **100 ms cumulative visible exposure**. Flow exposure uses
|
||||
10% of the smaller of page/viewport area and samples scroll positions plus a
|
||||
50 ms timer. This detects brief views; it does not assess reading speed.
|
||||
- Only actually observed pages count. A scrollbar/slider jump does not credit
|
||||
all pages between the starting position and destination.
|
||||
- Hidden covers/extras are excluded from the denominator.
|
||||
- Source versions have separate visit coverage; qualifying time combines into
|
||||
existing chapter totals without changing reading identities.
|
||||
- Below-60% visits remain bounded diagnostic drafts, excluded from Reading Story,
|
||||
first/last-read duration, reread counts and chapter headline totals.
|
||||
- New navigation starts a fresh visit. Old coverage does not qualify later picture
|
||||
lookups. Separate sub-60% visits are not combined. Back/forward-cache restoration
|
||||
continues the same visit.
|
||||
|
||||
Example: view 12 of 20 pages, then leave → all active time from that visit counts.
|
||||
Jump to page 19, dwell, leave → no new reading total. Historical times are retained;
|
||||
this update neither deletes old measurements nor invents times for missed reads.
|
||||
|
||||
Intent cannot be inferred perfectly: slowly searching most of a chapter can look
|
||||
like reading. This rule filters quick-search-and-one-picture visits without
|
||||
penalizing honest fast readers. Ordinary read/unread progress is independent.
|
||||
|
||||
## Pauses and display
|
||||
|
||||
Pause when hidden/unfocused; resume on reader input after returning. Pause after
|
||||
**30 seconds** without activity (updated at the user's request on 2026-10-06). The first loaded page can start
|
||||
without an extra click. Mouse/pen motion, wheel, touch movement, momentum scrolling
|
||||
and keyboard controls—including held Up/Down arrows—refresh reader activity.
|
||||
|
||||
Loading wait stays separate. An unloaded neighbour no longer pauses the clock if
|
||||
another visible image is readable. Failed/unloaded images do not earn coverage.
|
||||
An end card pauses time. Other reader tabs/devices coordinate to avoid accidental
|
||||
duplicate clocks; this is not an anti-cheat score.
|
||||
|
||||
The toolbar shows **this visit's** time. Its panel shows pages viewed, the 60%
|
||||
requirement, whether the visit counts, saved chapter total, loading wait and
|
||||
pause/save-pending state. “Verified”, “rejects” and “skimming” labels are removed.
|
||||
|
||||
## Reliable saves and compatibility
|
||||
|
||||
New clients POST cumulative snapshots to authenticated `/api/reading/visit` every
|
||||
10 seconds, on attention changes, and using `sendBeacon` (keepalive fallback) on
|
||||
`pagehide`. Final snapshots dispatch even with an ordinary request in flight.
|
||||
Visit ID plus increasing sequence makes retries/out-of-order packets idempotent.
|
||||
Delayed saves avoid the legacy 12-second packet truncation. Final saves don't
|
||||
reclaim the next chapter's lease, nor do stale pulses reclaim an exited lease.
|
||||
|
||||
First/last duration keeps growing after qualification until departure. Late finals
|
||||
update their own receipt, not a newer visit. The latest eight visit receipts are
|
||||
retained per chapter; extremely old delayed requests after eviction are outside
|
||||
this bounded retry window. There is no durable offline queue: closing an offline
|
||||
tab or a browser/OS crash can still lose undelivered time. “Save pending” is explicit.
|
||||
|
||||
Legacy `/api/reading/pulse` remains for already-open old readers. Existing records
|
||||
and totals are retained. Reload open readers for this policy. On code rollback the
|
||||
separate new endpoint returns 404 instead of interpreting cumulative totals as
|
||||
legacy deltas. Refresh open readers after rollback as well.
|
||||
|
||||
## Privacy and verification
|
||||
|
||||
Only chapter/version IDs, visit/sequence IDs, cumulative active/loading/input-type
|
||||
times, viewed page indexes and exit flags are sent. No coordinates, key contents,
|
||||
recordings, screenshots or raw event stream are saved. Exposure geometry stays
|
||||
local. History is private to the user and existing export/reset controls remain.
|
||||
|
||||
`test/reading-visits.test.js` covers thresholds, historical preservation, fast
|
||||
slices, lookup exclusion, attention/loading, late/duplicate snapshots and delayed
|
||||
delivery. `node tools/test-compose.js --reading-ui` adds actual headless Edge
|
||||
mobile-layout QA with disposable users/generated images/isolated storage. It does
|
||||
not observe production readers or query image hosters.
|
||||
@@ -0,0 +1,113 @@
|
||||
# Runtime resilience and agent operations
|
||||
|
||||
## Ownership and failure boundaries
|
||||
|
||||
```text
|
||||
Browser → Reader ──private API──→ Scraper → library (atomic chapter publication)
|
||||
│ ↑
|
||||
└──health──→ Sentinel ───┘ bounded checkpoint recovery only
|
||||
|
||||
Supervisor(reader) Supervisor(scraper) Supervisor(sentinel)
|
||||
↑
|
||||
Coding agent: inspect incidents → reproduce → patch → test → release
|
||||
```
|
||||
|
||||
The reader is the only writer of users/progress and the catalog. The scraper is the
|
||||
only queue owner and content publisher. Sentinel owns only observations and incidents.
|
||||
Each service's supervisor owns only that child process. The coding agent is an
|
||||
**escalation/maintenance operator**, not a necessary part of normal downloading.
|
||||
The recurring Codex check is an additional backstop, not the runtime watchdog.
|
||||
|
||||
## Automatic recovery implemented
|
||||
|
||||
| Failure | Action | Automatic limit |
|
||||
|---|---|---|
|
||||
| Process exit or repeated failed liveness checks | Supervisor restarts only that service, with backoff | Three failures per rolling hour; then parked for operator action |
|
||||
| Network/HTTP 5xx | Preserve checkpoint; open source circuit, wait five minutes, retry job | One job-level retry; request-level retries remain bounded |
|
||||
| Title unavailable/no English chapters | Record unavailable, continue unrelated work | No blind retry |
|
||||
| Parser/local adapter failure | Quarantine job, keep partial data, continue other titles | Two different affected jobs in an hour pause source as systemic failure |
|
||||
| Storage/permission failure | Pause all downloads and report | No automatic retry |
|
||||
| HTTP 401/403/429 or access challenge | Pause affected source lane | No automatic retry/bypass |
|
||||
| Active job with no progress for 15 minutes | Sentinel requests cooperative checkpoint-safe retry | One stall recovery; next stall quarantined |
|
||||
|
||||
The daemon remains authoritative: a recovery request includes the observed progress
|
||||
fingerprint and is rejected if the job has moved, paused, or is no longer stale.
|
||||
Sentinel also observes completed/page counts; a deliberately paused queue is not a
|
||||
stall. Timeouts bound source fetches to 60 seconds and metadata calls to 30 seconds.
|
||||
Publication remains one complete chapter at a time, on the same filesystem.
|
||||
No recovery action raises pacing, creates another worker for the same source,
|
||||
changes identities or deletes partial images.
|
||||
|
||||
The publisher retains validated pages in `.staging-mcscraper-resume-<sha256>` folders
|
||||
on pause/failure. It checks the full chapter/source/page-manifest fingerprint before
|
||||
reuse and revalidates each image; completed chapters still publish with one rename.
|
||||
A changed manifest creates separate staging rather than mixing releases. Old random
|
||||
stages remain untouched for manual review: they cannot be safely associated from
|
||||
page filenames alone. Hidden pending image files are never treated as completed pages.
|
||||
|
||||
`retry_wait`, `unavailable` and `quarantined` are explicit queue states in schema 4.
|
||||
`ready` is a healthy inspected job awaiting operator download confirmation, not a
|
||||
stall. Inspection shares the source lane; it never downloads images or bypasses a
|
||||
source circuit. Source coverage and job completion are separate observations.
|
||||
Legacy unavailable-title errors are migrated; only pauses solely attributable to
|
||||
these unavailable titles are released. Access and unknown failures remain paused.
|
||||
Nightly discovery avoids duplicating outstanding, unavailable or quarantined jobs;
|
||||
use an explicit operator retry after fixing their cause.
|
||||
|
||||
## Observability
|
||||
|
||||
**Settings → Service health** shows Sentinel status, active checkpoints, failed jobs
|
||||
and incident history. **Settings → Scraper** retains queue, pacing and discovery controls.
|
||||
Service errors are classified rather than buried in a single historical `lastError`.
|
||||
`npm run check` performs read-only checks. Sentinel's private, token-protected `/metrics`
|
||||
exports reader-up, scraper-up and open-incident gauges for a future Prometheus collector.
|
||||
It is not a public endpoint.
|
||||
|
||||
Scraper status is independent of expensive library scans. Concurrent admin/Sentinel
|
||||
probes share one background inventory refresh and receive the previous summary
|
||||
immediately. The dashboard summary refreshes at most once per minute unless a
|
||||
publication or metadata change invalidates it; duplicate-sensitive decisions still
|
||||
request a fresh inventory. `libraryStatus` reports ready/refreshing/stale/updated/error, and failed
|
||||
background refreshes do not spawn a new scan on every UI poll. Authoritative
|
||||
discovery waits for the current inventory; publication/metadata invalidation during
|
||||
a scan forces another scan before it can be used for those decisions. Cold start
|
||||
may temporarily show no library targets, but queue and rate controls remain usable.
|
||||
|
||||
Storage:
|
||||
|
||||
- `data/catalog.json`, `data/collection-covers/`: reader catalog and covers.
|
||||
- `data/scraper/queue.json`, `events.log`: queue, circuits, failures and incidents.
|
||||
- `data/sentinel/state.json`: Sentinel incidents and last observation.
|
||||
- `data/runtime-reader.json`, `data/scraper/runtime.json`, `data/sentinel/runtime.json`:
|
||||
supervisor status and persisted crash budget.
|
||||
- `data/scraper/supervisor.lock`, `daemon.lock`: native process ownership; transient.
|
||||
|
||||
Scraper/Sentinel/supervisor emit structured logs. Queue and Sentinel histories retain
|
||||
200 incidents; the UI shows a bounded subset. Docker logs are rotated. Native Windows
|
||||
supervisor logs remain append-only: archive them during stopped maintenance if needed.
|
||||
Never export user databases, tokens, cookies, signed query strings or unredacted environments.
|
||||
|
||||
## What self-repair intentionally does not mean
|
||||
|
||||
Sentinel cannot execute a shell, patch adapters, query image sources, access a Docker
|
||||
socket, or issue unrestricted restart commands. It cannot repair broken markup or
|
||||
solve source challenges. The agent handles those escalations in the repository:
|
||||
|
||||
1. Inspect one bounded incident and relevant checkpoint; preserve the active lane.
|
||||
2. Add a fixture reproducing the failure, patch the smallest adapter/logic surface.
|
||||
3. Run `npm test`; test UI/API in isolated data if affected.
|
||||
4. Build a traceable image, deploy checkpoint-safely, verify resumed progress.
|
||||
5. Record release, backup and rollback compatibility. Stop after repeated identical failures.
|
||||
|
||||
A VM-local monitor cannot alert on total VM/network failure. Before VM cutover choose
|
||||
an external health check and notification destination (e.g. an existing monitoring
|
||||
service and private ntfy/Gotify). No notification credentials or destination are invented.
|
||||
|
||||
## Crash-budget operation
|
||||
|
||||
A parked supervisor stays alive and reports `repair-required`; Docker's restart policy
|
||||
does not create an unlimited process restart storm. Its health check fails while the
|
||||
child is absent. Investigate first. Once repaired, restart the wrapper after its
|
||||
rolling one-hour budget has expired. For an intentional earlier reset, first stop the
|
||||
service, back up its `runtime*.json`, and clear only that service's `failures` array;
|
||||
never clear the queue or daemon checkpoint. Explicit deployments do not erase budgets.
|
||||
+37
-2
@@ -83,9 +83,13 @@
|
||||
```
|
||||
c<NNNN>[.<d>][ <title>][ [<Source>]] regular chapter
|
||||
x<NN>[ <title>][ [<Source>]] extra chapter outside the numbering
|
||||
special-mangadex-<chapter-uuid>[ <title>][ [<Source>]] unnumbered source release
|
||||
```
|
||||
- A **space** MUST follow the number when anything follows. `c0012_Title` sorts wrongly in file browsers; `c0012a` and `c0012Title` are not recognised at all.
|
||||
- Recommended check: `^(c\d{4}(\.\d*[1-9])?|x\d{2})( [^\[\]]+?)?( \[[^\[\]]+\])?$`
|
||||
- For the additive `special-mangadex-` form, validate the complete UUID using
|
||||
`chapter-identity.js`. Do not apply the numeric-only check above. The key is the
|
||||
lower-case source UUID prefix, not an invented numeric chapter. Books remain numeric.
|
||||
- **R-C2** Pad the chapter number's integer part to 4 digits. Never change the padding of an existing manga: chapter folder names are version ids.
|
||||
- **R-C3** Map upstream numbering like this:
|
||||
|
||||
@@ -102,6 +106,14 @@
|
||||
| Volume extras (afterword, sketches) | `x<NN>` in that volume's book | |
|
||||
| One-shot | its own manga (`v01/c0001`), or `x<NN>` inside an `x01 Oneshots` book of the related series | |
|
||||
| Negative or non-numeric only | `x<NN>` | Never invent a number that collides with a real one |
|
||||
| Unnumbered MangaDex release | `special-mangadex-<chapter-uuid>` | Stable upstream release identity; label Pilot, One-shot or Special; do not assign chapter 0/1 |
|
||||
|
||||
Unnumbered source-ID releases sort after numeric chapters. Their `chapter.json`
|
||||
stores `special:true`, `label`, `sourceId` and the original `sourceNumber` (usually
|
||||
null). They are not automatically skippable extras: an entire pilot may be story
|
||||
content. Keep different release UUIDs separate rather than guessing that they are
|
||||
the same logical chapter. Reimporting the same key is duplicate-safe. Existing
|
||||
numeric and `x` folder conventions and progress IDs MUST NOT be migrated or renamed.
|
||||
|
||||
- **R-C4** The title part of the folder name is for humans only.
|
||||
- The authoritative title is `chapter.json` → `title`, without the "Chapter 12:" prefix.
|
||||
@@ -174,7 +186,7 @@ All are UTF-8 JSON **without** BOM, pretty-printing optional.
|
||||
}
|
||||
```
|
||||
|
||||
- `direction`:
|
||||
- `direction` when an operator or reliable source explicitly provides the reading order:
|
||||
|
||||
| Upstream type | Value |
|
||||
|---------------|-------|
|
||||
@@ -186,6 +198,24 @@ All are UTF-8 JSON **without** BOM, pretty-printing optional.
|
||||
- Unknown fields are kept and ignored by the app. Keep your own state in one namespaced field such as `"x-scraper"`.
|
||||
- The whole file is shown to logged-in users. It MUST NOT contain secrets (tokens, cookies, credentials).
|
||||
- **Read-modify-write:** keep fields you don't own, because people may have edited them. Write atomically (R-W5).
|
||||
- **R-J4** The current adapters do not infer printed reading order from host,
|
||||
language, country, or catalog labels. They emit `rtl` with
|
||||
`"x-reading-direction":{"version":1,"value":"rtl","source":"fallback"}`.
|
||||
The publisher checks already downloaded image dimensions: at least three pages
|
||||
with height/width ≥ 4, at least 75% tall among known pages, consistent strip
|
||||
widths within 25%, and dimensions known for at least 75% of pages yield
|
||||
`direction: "ttb"` with `source: "detected"` and numeric evidence. A credits
|
||||
card can coexist with the strips. This detects layout, not content or language.
|
||||
Plain legacy directions and explicit `directionOverride` at any level are
|
||||
preserved across later imports. Do not bulk-convert old `rtl` values: a past
|
||||
scraper default and a human choice are indistinguishable. Use the reviewed
|
||||
migration tool on a named series after checking representative chapters.
|
||||
`node tools/review-reading-direction.js <absolute-library-manga-folder>
|
||||
<weebcentral-series-id> --chapter=c0056` is a read-only review. To apply a
|
||||
supported single-series correction, first pause and stop every writer and
|
||||
create a stopped-writer snapshot, then add `--apply --writers-stopped`.
|
||||
The tool verifies source identity, unchanged legacy RTL metadata, several
|
||||
chapter samples, and the requested chapter before an atomic metadata write.
|
||||
|
||||
- **R-J2** `chapter.json`, one per version folder:
|
||||
|
||||
@@ -240,7 +270,12 @@ MangaCircus can scan at any moment. Everything it can see must always be complet
|
||||
4. delete `.old-<random>`.
|
||||
The chapter is briefly absent, which is harmless: it won't be re-announced, and progress is kept.
|
||||
- **R-W5** Updating a JSON file: write `.<name>.tmp` in the same folder, then `rename` it over the target.
|
||||
- **R-W6** At start-up, delete your own leftover `.staging-*` / `.old-*` folders older than a few hours (from crashed runs).
|
||||
- **R-W6** Preserve incomplete staging folders: validated images are recovery assets,
|
||||
not disposable garbage. The managed publisher resumes a hidden folder only when
|
||||
its complete chapter/source/page-manifest fingerprint matches, and revalidates
|
||||
stored images. Legacy random stages lack this identity and require manual review;
|
||||
do not infer ownership or delete them merely because they are old. Completed
|
||||
replacement `.old-*` backups may be cleaned after successful publication.
|
||||
- **R-W7** On Windows, `rename` can fail with `EPERM`/`EBUSY` while another process reads the folder. Retry a few times with a short delay.
|
||||
- **R-W8** Files MUST be readable by the MangaCircus process. In Docker that is uid 1000, so use mode `0644` for files and `0755` for folders.
|
||||
- **R-W9** You MUST NOT write anything into MangaCircus's `DATA` folder (`db.json`, `scan-cache.json`).
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
# Managed scraper service
|
||||
|
||||
The scraper is an optional Node.js daemon that runs beside the MangaCircus reader.
|
||||
It owns the durable queue and is the only process allowed to run a queued archive.
|
||||
The browser never contacts it directly: authenticated admin routes in `server.js`
|
||||
proxy commands to its private HTTP API.
|
||||
|
||||
## Start and storage
|
||||
|
||||
```bash
|
||||
npm run start:all # reader + daemon under one local supervisor
|
||||
npm start # reader only
|
||||
npm run start:scraper # daemon only
|
||||
```
|
||||
|
||||
These commands are for native development only. The current Windows installation
|
||||
runs the three-service Docker Compose stack; the old Scheduled Task is disabled.
|
||||
Never start a native supervisor or daemon against its mounted state. Its observed
|
||||
ports, logs, persisted settings, restart procedure, and recovery snapshot are recorded in
|
||||
[`local-deployment.md`](local-deployment.md).
|
||||
|
||||
| Variable | Default | Purpose |
|
||||
|---|---|---|
|
||||
| `SCRAPER_HOST` | `127.0.0.1` | Daemon bind address |
|
||||
| `SCRAPER_PORT` | `8091` | Daemon port |
|
||||
| `SCRAPER_TOKEN` | empty | Bearer token; mandatory when the bind address is not loopback |
|
||||
| `SCRAPER_DATA` | `<DATA>/scraper` | Durable queue, lock, and event log |
|
||||
| `LIBRARY` | `./library` | Shared writable library for the daemon |
|
||||
| `SCRAPER_URL` | `http://127.0.0.1:8091` | Daemon URL used by the reader |
|
||||
|
||||
The authoritative queue is `data/scraper/queue.json` (currently schema version 4),
|
||||
written atomically. Individual
|
||||
archives retain their existing resumable checkpoints under
|
||||
`library/.mcscraper-work-<source-id>/job.json`. `events.log` is append-only.
|
||||
|
||||
On its first start, the daemon imports the old `.mcscraper-queue/queue.json` wishlist
|
||||
when present. Completed chapter ids in each archive checkpoint remain authoritative,
|
||||
so migration does not redownload published chapters.
|
||||
|
||||
## Lifecycle
|
||||
|
||||
The daemon owns one durable queue and runs at most one sequential worker per source. Different sources may run side by side, but two jobs from the same source never overlap:
|
||||
|
||||
1. Select the first `queued` item for every idle, unpaused source.
|
||||
2. Resolve its URL through the adapter registry in `scraper/adapters/index.js`.
|
||||
3. Run one adapter job per source with that source's request and chapter delays.
|
||||
4. For legacy in-progress checkpoints, continue publishing into their private work directory and expose the complete manga at the end.
|
||||
5. For every new job, stage one chapter beside its final book and expose that complete chapter immediately with one rename.
|
||||
6. Increment `publishedRevision` after every newly visible chapter and start the next item for that source when the selected scope is complete.
|
||||
|
||||
Legacy work can be converted without redownloading. While the queue is paused and no
|
||||
worker is active, `publishLegacyReady` validates every completed chapter, quarantines
|
||||
abandoned hidden partial stages, atomically exposes the accumulated manga folder, and
|
||||
changes its checkpoint to incremental mode. The resumed job then derives its remaining
|
||||
work from the visible library and publishes every later chapter immediately.
|
||||
|
||||
A source failure pauses only that source, allowing unrelated source workers to continue. Manual pause remains global. Request and chapter delays are configured independently for every installed host in **Settings → Scraper**. Both current adapters default to the fast profile: 1–4 seconds between sequential requests and 5–15 seconds between chapters.
|
||||
|
||||
The reader polls `publishedRevision` every 15 seconds and rescans after it increases.
|
||||
A source error pauses only that source lane for human review. Pause and shutdown requests
|
||||
are checked between requests; validated partial pages are retained in hidden,
|
||||
fingerprinted staging and reused only when the complete manifest still matches.
|
||||
|
||||
## Inspect, confirm, and interpret completion
|
||||
|
||||
The admin form uses **Inspect & queue** (`inspect:true`). This queues a catalog-only
|
||||
`kind:"inspection"` item on the normal sequential source lane, at the configured
|
||||
delays. No image or cover requests occur during inspection. Successful inspection
|
||||
sets `status:"ready"`, displays the canonical `sourceTitle` and source availability,
|
||||
and waits for **Confirm download**. Ready items survive daemon restarts and suppress
|
||||
duplicate nightly checks. Other jobs may proceed while confirmation is pending.
|
||||
Confirmation does not resume a deliberately paused queue or clear a source block.
|
||||
The download rediscovers the catalog; an edition title change requires re-inspection
|
||||
and confirmation. Direct API clients can still enqueue authorized downloads without
|
||||
inspection by omitting the flag. Existing download/discovery jobs are unchanged.
|
||||
|
||||
**Inspect again** operates on an inactive ready, completed, unavailable or quarantined
|
||||
item. It never overrides a source access block and does not fetch external-link pages.
|
||||
Old MangaDex unavailable messages are marked as needing re-inspection, not silently
|
||||
retried. No remote diagnosis is inferred from that old numeric-only adapter result.
|
||||
If the item was a completed download, inspection compares its selected chapters with
|
||||
the shared library inventory. An already-present selection remains **Complete** with
|
||||
its original completion and publication counts; it does not become an awaiting-download
|
||||
item. Newly available chapters instead remain **Ready** for explicit confirmation.
|
||||
WeebCentral catalog inspection accepts numbered Chapter, Episode, Rating, Night,
|
||||
Log and Act labels. If chapter links exist but none can be numbered, the parser
|
||||
reports a label-format problem; an actually empty catalog is reported separately.
|
||||
|
||||
`complete` means the selected **job** finished, not that a series is fully loaded.
|
||||
`publishedCount` counts chapters newly exposed by that run; `selectedCount` is its
|
||||
remaining selected scope. Resume may reduce those counts because the inventory skips
|
||||
chapters already published. MangaDex availability records hosted release/selection
|
||||
counts, unnumbered release counts, metadata's last chapter and the highest hosted
|
||||
number. A lower hosted maximum proves partial coverage; an equal maximum does **not**
|
||||
prove completeness. Other cases say completeness is unverified. The last chapter
|
||||
number is never used as a chapter count.
|
||||
|
||||
Empty English hosted catalogs distinguish no English releases listed, English
|
||||
external-link-only releases, and no hosted English releases with pages. External
|
||||
availability is checked through a separate, paced `includeExternalUrl=1` feed request
|
||||
only when hosted selections are empty; downloads use `includeExternalUrl=0`.
|
||||
|
||||
## Queue scopes and duplicate policy
|
||||
|
||||
Every new queue item has its own UUID plus the stable upstream `seriesId`. This allows
|
||||
several jobs for one manga (for example volume 2 followed by volume 3) without sharing
|
||||
queue identity. Its `scope` is one of:
|
||||
|
||||
| Mode | Meaning |
|
||||
|---|---|
|
||||
| `missing` | Every eligible chapter absent from the library |
|
||||
| `latest` | The newest eligible missing chapter |
|
||||
| `last` / `first` | The last/first `count` eligible missing chapters |
|
||||
| `chapter` | One numeric chapter |
|
||||
| `after` | Every numeric chapter after the supplied chapter |
|
||||
| `volume` | Chapters whose source catalog explicitly reports that volume |
|
||||
| `ids` | Internal exact source chapter ids produced by nightly discovery |
|
||||
|
||||
Unnumbered MangaDex releases use `special-mangadex-<chapter-uuid>` as their logical
|
||||
key, never an invented chapter zero or one. Numbered release deduplication is unchanged;
|
||||
unnumbered releases stay distinct unless their source UUID is identical. Two pilot
|
||||
translations therefore remain two labeled, group-credited releases, not a guessed
|
||||
numbered chapter or silently merged translation. Numeric `chapter`/`after` scopes
|
||||
exclude them; missing, exact-ID and applicable volume scopes include them. Catalog
|
||||
ordering puts numbered chapters first and source-ID-ordered specials afterward;
|
||||
first/last/latest operate on that ordering, not on publication timestamps.
|
||||
|
||||
`existingPolicy:"skip-any"` skips a chapter when any version already exists.
|
||||
`existingPolicy:"add-source"` permits another host's version but still skips a version
|
||||
already present from the selected host. Existing version folders are never overwritten.
|
||||
|
||||
Before downloading, the scraper reads a lightweight library inventory. Work identity is
|
||||
matched by exact source id, exact source URL, explicit library manga id, or verified
|
||||
AniList id—never by fuzzy title. The filesystem library remains authoritative: a stale
|
||||
checkpoint cannot make a missing published chapter look complete.
|
||||
|
||||
## Nightly discovery
|
||||
|
||||
`manga.json` stores the admin override at `x-scraper.autoUpdate`. When it is absent,
|
||||
`status:"ongoing"` defaults to enabled and other statuses default to disabled. At the
|
||||
configured local `autoUpdateTime`, the daemon queues one catalog-only discovery item for
|
||||
the preferred supported source of every enabled manga. Discovery shares the normal
|
||||
sequential source lane and pacing. If strictly newer eligible chapters are found, that
|
||||
same item is promoted to a download job containing their source ids. A successful
|
||||
no-change check records `lastCheckedAt` and its result in the per-manga/source
|
||||
update summary, appends history, and retires the discovery queue row without page
|
||||
requests. `lastDownloadedAt` is recorded when a chapter is published or a download
|
||||
job completes. These summaries persist in `queue.json` as `updateChecks`; on load,
|
||||
legacy completed no-change discovery rows migrate into the summary and are retired.
|
||||
Failed checks remain visible for attention. Duplicate pending checks for the same
|
||||
source series are suppressed.
|
||||
Only numbered chapters strictly newer than the library's highest regular chapter,
|
||||
plus unseen source-ID specials, are considered updates. Historical numbered gaps
|
||||
are never backfilled by the nightly task; use an explicit queue
|
||||
scope when a gap is intentional work.
|
||||
|
||||
## Private daemon API
|
||||
|
||||
All responses are JSON. When `SCRAPER_TOKEN` is set, send
|
||||
`Authorization: Bearer <token>`.
|
||||
|
||||
| Method | Path | Body / result |
|
||||
|---|---|---|
|
||||
| `GET` | `/health` | Basic liveness |
|
||||
| `GET` | `/api/state` | Queue, all active source checkpoints, settings, errors, history, adapters |
|
||||
| `POST` | `/api/queue` | `{url,title?,metadataSearch?,scope?,existingPolicy?,libraryMangaId?,inspect?}` |
|
||||
| `POST` | `/api/queue/remove` | `{id}`; active downloads cannot be removed |
|
||||
| `POST` | `/api/queue/clear-completed` | `{source?}`; remove inactive successful download job rows only; preserve library, history, checkpoints and other statuses |
|
||||
| `POST` | `/api/queue/reorder` | `{ids}` containing every queue id exactly once |
|
||||
| `POST` | `/api/control` | `{action:"pause"|"resume"|"retry"|"inspect"|"confirm"|"skip",id?}` |
|
||||
| `POST` | `/api/settings` | `sourceRates.{adapter}.{minMs,maxMs,chapterMinMs,chapterMaxMs}` plus `autoUpdateEnabled` and local `autoUpdateTime` (`HH:MM`) |
|
||||
| `POST` | `/api/discovery/run` | Queue an immediate update check for every enabled manga |
|
||||
| `POST` | `/api/library/auto-update` | `{id,enabled}` updates `manga.json` atomically |
|
||||
| `POST` | `/api/legacy/publish-ready` | Paused-only validation/publication of completed legacy work; converts its checkpoint to incremental mode |
|
||||
|
||||
Legacy top-level rate fields remain a fallback for adapters without a per-source entry. Settings also accept `autoUpdateEnabled` and a local `autoUpdateTime` in `HH:MM` form.
|
||||
|
||||
The reader exposes only `GET/POST /api/admin/scraper`, protected by the existing
|
||||
admin session. It maps a small command allowlist to the private API and never returns
|
||||
the shared token.
|
||||
|
||||
## Adapter contract
|
||||
|
||||
Each source plugin is registered in `scraper/adapters/index.js` with:
|
||||
|
||||
```js
|
||||
{
|
||||
id: 'host-id',
|
||||
label: 'Human name',
|
||||
match(url) { /* return {source,id,title,url} or null */ },
|
||||
async discover(item, { settings, shouldStop, onEvent }) { /* catalog only */ },
|
||||
async run(item, {
|
||||
libraryRoot, settings, shouldStop, onPublished, onCatalog
|
||||
}) { /* publish selected chapters and return counts/final path */ }
|
||||
}
|
||||
```
|
||||
|
||||
Adapters discover host-specific catalogs and pages, but reuse the central pacing,
|
||||
checkpoint, metadata, and atomic publisher modules. They must stop on HTTP 401, 403,
|
||||
429, CAPTCHA, challenge, or access-denied responses. Never add protection bypasses.
|
||||
|
||||
`onCatalog(catalog)` persists source title and optional availability before any page
|
||||
acquisition. The MangaDex adapter accepts only explicit title UUID URLs, verifies
|
||||
AniList cross-references, selects one English release per numeric chapter, preserves
|
||||
unnumbered source identities, credits groups and downloads sequential full-resolution
|
||||
MangaDex@Home pages. Special-only editions do not use fuzzy AniList lookup or AniList
|
||||
fallback library matching; their canonical source title names new folders. Explicit
|
||||
library targets remain operator choices; existing folders are never renamed.
|
||||
|
||||
## Admin UI
|
||||
|
||||
**Settings → Admin → Scraper** polls the authenticated proxy every three seconds
|
||||
only while that section is active. Activity, Queue, Updates and Sources are
|
||||
separate local tabs with a shared source filter. The section shows:
|
||||
|
||||
- every active source, manga, book/season, chapter, page, and overall progress;
|
||||
- source warnings and fatal errors;
|
||||
- queue ordering and removal controls;
|
||||
- add-by-URL, pause, resume, and retry controls;
|
||||
- queue scope, exact chapter/count/volume, target manga, and alternate-source controls;
|
||||
- per-manga auto-update checkboxes, nightly time, and a manual **Check now** action;
|
||||
- last checked and last downloaded times per manga/source, with old check events
|
||||
collapsed into recent history instead of occupying queue space;
|
||||
- a collapsed completed-jobs section with confirmation before bulk cleanup;
|
||||
- persistent image and chapter delay ranges.
|
||||
|
||||
Rate changes apply on the next manga. To apply them to the active manga, pause and
|
||||
resume it; the current chapter restarts safely if it was incomplete.
|
||||
@@ -0,0 +1,81 @@
|
||||
# Settings workspace
|
||||
|
||||
The settings page has two menu levels: ownership (User/Admin), then a focused
|
||||
section. Only one section is shown at a time. Scraper tabs are local controls,
|
||||
not a third global menu level.
|
||||
|
||||
```text
|
||||
Settings
|
||||
User
|
||||
Reading
|
||||
Reading time & privacy
|
||||
Notifications
|
||||
Account
|
||||
Admin
|
||||
Scraper
|
||||
[Activity | Queue | Updates | Sources]
|
||||
Server & library
|
||||
Users & access
|
||||
Collections & metadata
|
||||
Service health
|
||||
```
|
||||
|
||||
Desktop uses a sticky sidebar; mobile uses a wrapping menu above the content.
|
||||
Active sections have a visual marker and `aria-current`. Navigation uses real
|
||||
links, supports keyboard use and browser back/forward, and focuses the section
|
||||
heading when switching within Settings.
|
||||
|
||||
## Routes and ownership
|
||||
|
||||
Personal routes: `settings.html#user/reading`, `#user/reading-time`,
|
||||
`#user/notifications`, `#user/account`.
|
||||
|
||||
Admin routes: `settings.html#admin/scraper`, `#admin/library`, `#admin/users`.
|
||||
Scraper deep links append `/activity`, `/queue`, `/updates` or `/sources`.
|
||||
The old `/hosters` link opens Sources. The scraper default is Activity.
|
||||
Existing `collections.html` and `health.html` retain their URLs but share the
|
||||
same menu. `#admin/collections` and `#admin/health` redirect to those pages.
|
||||
Unknown routes default to Reading. Non-admin accounts never render the Admin menu
|
||||
or load admin data from Settings; admin hash routes normalize to Reading.
|
||||
Server-side authorization remains authoritative and still rejects non-admin API calls.
|
||||
|
||||
`public/settings-nav.js` defines the shared menu and route model. `settings.js`
|
||||
preserves personal form nodes across section changes and loads admin sections lazily.
|
||||
Scraper polling runs only while its section is active. Draft inputs survive polling
|
||||
and tab switches; successful commands reload state. Command failures are visible
|
||||
without deleting the user's draft. Source chips filter the four scraper views
|
||||
without changing queue ownership or per-source pacing.
|
||||
The Add manga, Completed downloads and Recent check activity disclosure boxes
|
||||
also retain their open/closed state across polling and command redraws; a page
|
||||
reload still starts them collapsed.
|
||||
|
||||
Activity shows active jobs, source attention, and the next queued items. Queue
|
||||
keeps reorder/inspect/confirm/remove controls, scopes, and an Add manga form;
|
||||
completed jobs are collapsed by default. **Clear completed downloads** removes
|
||||
only inactive, successful download job rows, optionally limited to the selected
|
||||
source. It never deletes library content, checkpoints, history or other statuses.
|
||||
Updates shows the nightly schedule, per-manga switches, the latest check and
|
||||
download times by manga/source, and a collapsed recent-check log. Successful
|
||||
no-change nightly checks update this summary and leave the queue. Sources owns
|
||||
the per-hoster request and chapter delay settings.
|
||||
|
||||
Reading-time collection controls, export and deletion live under Reading time &
|
||||
privacy; reader layout controls stay under Reading. Rescan controls and user
|
||||
administration are separate. Server & library describes existing maintenance,
|
||||
not an editor for Docker environment variables or an unrestricted restart console.
|
||||
|
||||
## Verification
|
||||
|
||||
`npm test` covers route defaults, deep links, ownership and existing service/API
|
||||
behavior. `node tools/test-compose.js --settings-ui` adds optional Playwright
|
||||
headless browser QA against the disposable Compose fixture, never the real user
|
||||
database or queue. Make Playwright available on the development machine; Windows
|
||||
uses installed headless Edge by default, overridable with `QA_BROWSER_CHANNEL`.
|
||||
The application itself remains dependency-free.
|
||||
|
||||
The UI check verifies desktop and 390×844 mobile navigation, form saves, draft
|
||||
retention, tab visibility, back/forward, stopping hidden scraper polling, shared
|
||||
collections/health navigation, non-admin denial and horizontal overflow.
|
||||
`--queue-ui` additionally checks the source filter, edition confirmation, add form,
|
||||
completed cleanup and mobile layout. Screenshots are generated under ignored
|
||||
`data/qa/settings/` and `data/qa/queue/`. No image sources are contacted.
|
||||
+9
-24
@@ -7,30 +7,14 @@
|
||||
* Image sizes are cached per chapter folder and only re-read when the folder's mtime changes.
|
||||
*/
|
||||
const fsp = require('fs').promises, path = require('path');
|
||||
const { parseSpecialFolder, parseKey } = require('./chapter-identity');
|
||||
const { resolveReadingDirection } = require('./reading-direction');
|
||||
|
||||
const IMG = /\.(jpe?g|png|webp|gif|avif|bmp)$/i;
|
||||
const COVER = /^cover\.(jpe?g|png|webp|gif|avif|bmp)$/i;
|
||||
const KEY = /^([a-z]*) ?(\d+(?:\.\d+)?)(?=$|[\s_\-[(])[\s_-]*(.*)$/i;
|
||||
const DIRECTIONS = { rtl: 'rtl', ltr: 'ltr', ttb: 'ttb', vertical: 'ttb', webtoon: 'ttb' };
|
||||
const cmp = new Intl.Collator('en', { numeric: true, sensitivity: 'base' }).compare;
|
||||
const str = x => (typeof x === 'string' ? x.trim() : '');
|
||||
const order = (a, b) => a.x - b.x || a.num - b.num;
|
||||
|
||||
// "c0002.5 Tea Time [MangaDex]" -> { key: '2.5', x: false, num: 2.5, title: 'Tea Time', src: 'MangaDex' }
|
||||
function parseKey(name) {
|
||||
const m = KEY.exec(name);
|
||||
if (!m) return null;
|
||||
const x = m[1].toLowerCase() === 'x', num = parseFloat(m[2]);
|
||||
let title = m[3].trim(), src = '';
|
||||
const b = /\s*\[([^\]]*)\]$/.exec(title);
|
||||
if (b) { src = b[1].trim(); title = title.slice(0, b.index).trim(); }
|
||||
return { key: (x ? 'x' : '') + num, x, num, title: title.replace(/^[-:.]\s*/, ''), src };
|
||||
}
|
||||
|
||||
function direction(...metas) {
|
||||
for (const m of metas) if (DIRECTIONS[str(m.direction).toLowerCase()]) return DIRECTIONS[str(m.direction).toLowerCase()];
|
||||
return 'rtl';
|
||||
}
|
||||
const order = (a, b) => Number(!!a.special) - Number(!!b.special) || a.x - b.x || a.num - b.num || cmp(a.key, b.key);
|
||||
|
||||
async function readJson(file) {
|
||||
try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^/, '')); }
|
||||
@@ -123,7 +107,7 @@ const coverIn = (files, rel) => { const f = files.find(f => COVER.test(f)); retu
|
||||
async function scanManga(root, name, oldCache, cache) {
|
||||
const { dirs, files } = await list(path.join(root, name));
|
||||
const meta = await readJson(path.join(root, name, 'manga.json'));
|
||||
const m = { id: name, title: str(meta.title) || name, meta, direction: direction(meta), cover: coverIn(files, name), books: [], chapters: [], chapterById: new Map(), updated: 0 };
|
||||
const m = { id: name, title: str(meta.title) || name, meta, direction: resolveReadingDirection([meta]).value, cover: coverIn(files, name), books: [], chapters: [], chapterById: new Map(), updated: 0 };
|
||||
const bookKeys = new Set();
|
||||
for (const bName of dirs) {
|
||||
const bk = parseKey(bName);
|
||||
@@ -138,14 +122,15 @@ async function scanManga(root, name, oldCache, cache) {
|
||||
};
|
||||
const byKey = new Map();
|
||||
for (const cName of bl.dirs) {
|
||||
const ck = parseKey(cName);
|
||||
const ck = parseSpecialFolder(cName) || parseKey(cName);
|
||||
if (!ck) { console.warn(`[scan] ignoring "${bRel}/${cName}": chapter folders must be named <letters><number>[ title], e.g. "c0001 The Seed"`); continue; }
|
||||
const v = await scanVersion(root, bRel + '/' + cName, ck, oldCache, cache);
|
||||
if (!v.pages.length) continue;
|
||||
v.direction = direction(v.meta, bMeta, meta);
|
||||
v.directionInfo = resolveReadingDirection([v.meta, bMeta, meta], v.pages);
|
||||
v.direction = v.directionInfo.value;
|
||||
let ch = byKey.get(ck.key);
|
||||
if (!ch) {
|
||||
ch = { id: book.key + '/' + ck.key, book: book.key, key: ck.key, num: ck.num, x: ck.x, extra: book.extra || ck.x, label: (ck.x ? 'Extra ' : 'Ch. ') + ck.num, title: '', versions: [] };
|
||||
ch = { id: book.key + '/' + ck.key, book: book.key, key: ck.key, num: ck.num, x: ck.x, special: !!ck.special, extra: book.extra || ck.x, label: ck.special ? str(v.meta.label) || 'Special' : (ck.x ? 'Extra ' : 'Ch. ') + ck.num, title: '', versions: [] };
|
||||
byKey.set(ck.key, ch);
|
||||
}
|
||||
if (v.meta.extra === true) ch.extra = true;
|
||||
@@ -179,4 +164,4 @@ async function scanLibrary(root, oldCache = {}) {
|
||||
return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache };
|
||||
}
|
||||
|
||||
module.exports = { scanLibrary, parseKey, sizeFromBuffer };
|
||||
module.exports = { scanLibrary, parseKey, sizeFromBuffer, imageSize };
|
||||
Generated
+65
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"name": "mangacircus",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "mangacircus",
|
||||
"version": "1.0.0",
|
||||
"devDependencies": {
|
||||
"playwright": "1.62.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
|
||||
"integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.62.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
|
||||
"integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+18
-2
@@ -2,13 +2,29 @@
|
||||
"name": "mangacircus",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "No-frills self-hosted manga reader. No dependencies.",
|
||||
"description": "No-frills self-hosted manga reader with no runtime dependencies.",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js",
|
||||
"demo": "node tools/make-dummy.js library"
|
||||
"start:scraper": "node scraper/daemon.js",
|
||||
"start:all": "node tools/start-all.js",
|
||||
"start:sentinel": "node sentinel/daemon.js",
|
||||
"check": "node tools/check.js",
|
||||
"check:compose": "node tools/compose-control.js check",
|
||||
"test:runtime": "node --test sentinel/test/*.test.js",
|
||||
"test:compose": "node tools/test-compose.js",
|
||||
"test:ui": "node tools/test-compose.js --settings-ui --queue-ui --collections-ui --reading-ui",
|
||||
"test:direction-ui": "node tools/test-compose.js --direction-ui",
|
||||
"demo": "node tools/make-dummy.js library",
|
||||
"scrape": "node scraper/cli.js",
|
||||
"test": "npm run test:reader && npm run test:scraper && npm run test:runtime",
|
||||
"test:reader": "node --test test/*.test.js",
|
||||
"test:scraper": "node --test scraper/test/*.test.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"devDependencies": {
|
||||
"playwright": "1.62.1"
|
||||
}
|
||||
}
|
||||
+196
-8
@@ -20,6 +20,7 @@ h1 { font-size: 26px; line-height: 1.2; margin: 0 0 6px; }
|
||||
}
|
||||
.btn:hover { background: var(--card2); }
|
||||
.btn.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
|
||||
.btn.danger { border-color: #8c3f48; color: #ff9aa7; }
|
||||
.btn.on { border-color: var(--accent); color: #ffb3bd; }
|
||||
.btn.small { min-height: 28px; padding: 2px 9px; font-size: 13px; }
|
||||
.btn.icon { padding: 6px 9px; }
|
||||
@@ -48,8 +49,39 @@ label { display: block; }
|
||||
main { max-width: 1200px; margin: 0 auto; padding: 16px; }
|
||||
|
||||
/* library */
|
||||
.toolbar { display: flex; gap: 8px; margin-bottom: 16px; }
|
||||
.toolbar input { flex: 1; min-width: 0; }
|
||||
.reading-summary {
|
||||
display: grid; grid-template-columns: minmax(150px, .7fr) minmax(220px, 1fr) auto; align-items: center; gap: 12px 22px;
|
||||
padding: 14px 16px; margin-bottom: 16px; border: 1px solid color-mix(in srgb, var(--accent) 45%, var(--line));
|
||||
border-radius: 12px; background: linear-gradient(120deg, color-mix(in srgb, var(--accent) 12%, var(--card)), var(--card));
|
||||
}
|
||||
.collection-mosaic { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.collection-mosaic img { width: 100%; height: 100%; object-fit: cover; min-height: 0; }
|
||||
.collection-count { display: inline-block; background: var(--card2); padding: 2px 6px; border-radius: 5px; font-size: 12px; }
|
||||
.collection-heading { grid-column: 1 / -1; margin: 12px 0 0; }
|
||||
.collection-member { display: flex; gap: 16px; align-items: center; padding: 10px; border-bottom: 1px solid var(--line); text-decoration: none; }
|
||||
.collection-member img { width: 55px; height: 80px; object-fit: cover; }
|
||||
.collection-editor > div { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
.collection-editor input[type=number] { width: 75px; }
|
||||
.catalog-json { width: 100%; min-height: 350px; font: 13px/1.4 monospace; }
|
||||
.card > textarea { display: block; width: 100%; margin: 10px 0; }
|
||||
.reading-summary-title { display: flex; gap: 8px; align-items: center; }
|
||||
.reading-summary-stats { display: flex; flex-wrap: wrap; align-items: baseline; gap: 4px 7px; }
|
||||
.reading-summary-stats strong { font-size: 18px; }
|
||||
.reading-summary-links { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 5px 14px; font-size: 13px; }
|
||||
.story-pill { padding: 1px 7px; border-radius: 10px; background: var(--card2); color: var(--muted); font-size: 12px; }
|
||||
.library-controls { margin-bottom: 18px; }
|
||||
.toolbar { display: flex; gap: 8px; margin-bottom: 10px; }
|
||||
.toolbar input { flex: 1; min-width: 180px; }
|
||||
.filter-group { border-top: 1px solid var(--line); padding-top: 10px; }
|
||||
.filter-heading { display: flex; align-items: baseline; gap: 10px; margin-bottom: 7px; }
|
||||
.tag-filter, .volume-filter { display: flex; gap: 7px; overflow-x: auto; padding: 1px 1px 7px; scrollbar-width: thin; }
|
||||
.filter-chip {
|
||||
flex: none; border: 1px solid var(--line); border-radius: 14px; padding: 3px 10px; background: var(--card); color: var(--fg);
|
||||
font: inherit; font-size: 13px; cursor: pointer; white-space: nowrap;
|
||||
}
|
||||
.filter-chip:hover { background: var(--card2); }
|
||||
.filter-chip.on { border-color: var(--accent); background: color-mix(in srgb, var(--accent) 22%, var(--card)); color: #ffd8de; }
|
||||
.result-count { margin-top: 6px; }
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); gap: 22px 16px; }
|
||||
.tile { position: relative; display: block; text-decoration: none; }
|
||||
.tile .cv { aspect-ratio: 2 / 3; border-radius: 8px; overflow: hidden; background: var(--card); box-shadow: 0 3px 14px rgba(0, 0, 0, .45); }
|
||||
@@ -63,8 +95,13 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; }
|
||||
.empty { text-align: center; padding: 60px 10px; }
|
||||
@media (max-width: 560px) {
|
||||
main { padding: 12px; }
|
||||
.toolbar { flex-wrap: wrap; }
|
||||
.toolbar input { flex-basis: 100%; }
|
||||
.toolbar select { flex: 1; }
|
||||
.grid { grid-template-columns: repeat(2, 1fr); gap: 16px 12px; }
|
||||
.nav { padding-left: 12px; padding-right: 12px; }
|
||||
.reading-summary { grid-template-columns: 1fr; gap: 5px; }
|
||||
.reading-summary-links { justify-content: flex-start; }
|
||||
}
|
||||
|
||||
/* manga page */
|
||||
@@ -73,26 +110,76 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; }
|
||||
.hero .info { min-width: 0; }
|
||||
.chips { display: flex; flex-wrap: wrap; gap: 6px; margin: 8px 0; }
|
||||
.chip { background: var(--card2); border-radius: 12px; padding: 1px 9px; font-size: 13px; }
|
||||
.chip-link { text-decoration: none; border: 1px solid transparent; }
|
||||
.chip-link:hover { border-color: var(--accent); color: #ffd8de; }
|
||||
.desc { white-space: pre-line; margin: 10px 0; }
|
||||
.actions { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 12px; }
|
||||
.sources a { margin-right: 10px; }
|
||||
.metadata.card { max-width: none; }
|
||||
.reading-story.card { max-width: none; padding: 0; overflow: hidden; }
|
||||
.reading-story-summary {
|
||||
display: flex; justify-content: space-between; align-items: center; gap: 14px; padding: 14px 16px; cursor: pointer;
|
||||
background: linear-gradient(120deg, color-mix(in srgb, var(--accent) 12%, var(--card)), var(--card));
|
||||
}
|
||||
.reading-story[open] > .reading-story-summary { border-bottom: 1px solid var(--line); }
|
||||
.story-stats { display: grid; grid-template-columns: repeat(4, minmax(110px, 1fr)); gap: 10px; padding: 14px 16px 8px; }
|
||||
.story-stat { min-width: 0; }
|
||||
.story-stat strong, .story-stat span { display: block; }
|
||||
.story-stat strong { font-size: 19px; font-variant-numeric: tabular-nums; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.story-last { margin: 4px 16px 16px; }
|
||||
.story-rhythm-head { display: flex; align-items: baseline; gap: 9px; margin: 0 16px 7px; }
|
||||
.story-rhythm {
|
||||
display: flex; align-items: flex-end; gap: 3px; min-height: 68px; margin: 0 16px; padding: 6px 3px 3px;
|
||||
overflow-x: auto; border-bottom: 1px solid var(--line); scrollbar-width: thin;
|
||||
}
|
||||
.rhythm-bar { flex: 0 0 7px; min-height: 3px; border-radius: 3px 3px 1px 1px; background: var(--accent); opacity: .86; }
|
||||
.rhythm-bar:hover, .rhythm-bar:focus { opacity: 1; outline: 2px solid #fff; outline-offset: 1px; }
|
||||
.rhythm-bar.empty { height: 3px; background: var(--line); opacity: 1; }
|
||||
.story-actions { display: flex; justify-content: space-between; align-items: center; gap: 12px; padding: 12px 16px 14px; }
|
||||
.metadata-summary { display: flex; align-items: center; justify-content: space-between; gap: 12px; cursor: pointer; list-style-position: outside; }
|
||||
.metadata.card .metadata-summary h2 { display: inline; margin: 0; }
|
||||
.metadata-summary::after { content: 'Show'; color: var(--muted); font-size: 13px; font-weight: 400; }
|
||||
.metadata[open] > .metadata-summary { margin-bottom: 12px; }
|
||||
.metadata[open] > .metadata-summary::after { content: 'Hide'; }
|
||||
.meta-grid { display: grid; grid-template-columns: minmax(130px, max-content) minmax(0, 1fr); gap: 8px 18px; margin: 0; }
|
||||
.meta-grid dt { color: var(--muted); }
|
||||
.meta-grid dd { min-width: 0; margin: 0; overflow-wrap: anywhere; }
|
||||
.meta-tags, .meta-links { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 10px; }
|
||||
.raw-meta { margin-top: 16px; }
|
||||
.raw-meta summary { cursor: pointer; color: var(--muted); }
|
||||
.raw-meta pre { max-height: 440px; overflow: auto; padding: 12px; border-radius: 8px; background: var(--bg); color: var(--muted); font-size: 12px; white-space: pre-wrap; overflow-wrap: anywhere; }
|
||||
@media (max-width: 640px) {
|
||||
.hero { flex-direction: column; align-items: center; text-align: center; }
|
||||
.hero > img { width: 180px; }
|
||||
.chips, .actions { justify-content: center; }
|
||||
.meta-grid { grid-template-columns: 1fr; gap: 2px; }
|
||||
.meta-grid dd { margin-bottom: 9px; }
|
||||
.story-stats { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.story-actions { align-items: flex-start; flex-direction: column; }
|
||||
}
|
||||
.chapters { border-top: 1px solid var(--line); }
|
||||
.chapter-section { margin-top: 24px; }
|
||||
.chapter-heading { display: flex; align-items: baseline; gap: 10px; }
|
||||
.chapter-heading h2 { margin: 0; }
|
||||
.volume-filter { margin: 9px 0 6px; max-width: 100%; }
|
||||
.chapter-list-wrap { max-width: 100%; overflow-x: auto; }
|
||||
.chapters { display: inline-block; width: max-content; min-width: min(440px, 100%); max-width: 100%; border-top: 1px solid var(--line); }
|
||||
.book-sep { color: var(--muted); font-size: 12px; text-transform: uppercase; letter-spacing: .08em; padding: 16px 4px 6px; }
|
||||
.ch { display: flex; align-items: center; gap: 8px; border-bottom: 1px solid var(--line); }
|
||||
.ch-main { flex: 1; min-width: 0; display: flex; gap: 10px; align-items: baseline; padding: 12px 4px; text-decoration: none; }
|
||||
.chapter-entry { width: 100%; }
|
||||
.ch { display: flex; width: 100%; max-width: 100%; align-items: center; gap: 6px; border-bottom: 1px solid var(--line); white-space: nowrap; }
|
||||
.ch-main { flex: 1 1 auto; min-width: 0; display: grid; grid-template-columns: 76px 52px minmax(70px, 1fr) max-content; gap: 8px; align-items: baseline; padding: 6px 4px; text-decoration: none; }
|
||||
.ch-main.has-time { grid-template-columns: 76px 52px 72px minmax(70px, 1fr) max-content; }
|
||||
.ch-main:hover .ch-title { text-decoration: underline; }
|
||||
.ch-label { font-weight: 650; flex: none; }
|
||||
.ch-label { min-width: 0; overflow: hidden; text-overflow: ellipsis; font-weight: 650; font-variant-numeric: tabular-nums; }
|
||||
.ch-pages { min-width: 0; overflow: hidden; color: var(--muted); font-size: 13px; font-variant-numeric: tabular-nums; text-align: left; white-space: nowrap; }
|
||||
.ch-time { min-width: 0; overflow: hidden; color: #c9a7ad; font-size: 13px; font-variant-numeric: tabular-nums; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.ch-title { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.ch.read .ch-main { color: var(--muted); }
|
||||
.tag { font-size: 11px; border: 1px solid var(--line); border-radius: 6px; padding: 0 5px; color: var(--muted); flex: none; }
|
||||
.state { color: var(--muted); font-size: 13px; background: none; border: 0; cursor: pointer; padding: 6px; font: inherit; font-size: 13px; }
|
||||
.state.done { color: #7fd3a0; }
|
||||
.versions { background: var(--card); border-radius: 8px; margin: 4px 0 10px; padding: 4px 10px; }
|
||||
.ch > .btn.small { min-height: 24px; padding: 1px 7px; }
|
||||
.ch > .state { padding: 3px 4px; }
|
||||
.versions { width: 100%; background: var(--card); border-radius: 8px; margin: 4px 0 10px; padding: 4px 10px; }
|
||||
.ver { display: flex; align-items: center; gap: 10px; padding: 7px 0; flex-wrap: wrap; }
|
||||
.ver + .ver { border-top: 1px solid var(--line); }
|
||||
.ver .src { flex: 1; min-width: 140px; }
|
||||
@@ -100,8 +187,71 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; }
|
||||
.vote button { background: var(--card2); border: 1px solid var(--line); color: var(--fg); border-radius: 6px; width: 30px; height: 28px; cursor: pointer; }
|
||||
.vote button.on { background: var(--accent); border-color: var(--accent); color: #fff; }
|
||||
.vote b { min-width: 22px; text-align: center; }
|
||||
.chapter-empty { margin: 16px 0; }
|
||||
@media (max-width: 640px) {
|
||||
.chapters { display: block; width: 100%; min-width: 0; }
|
||||
.ch { flex-wrap: nowrap; gap: 4px; overflow: hidden; }
|
||||
.ch-main { grid-template-columns: 64px 44px minmax(0, 1fr) max-content; gap: 6px; }
|
||||
.ch-main.has-time { grid-template-columns: 64px 44px 62px minmax(0, 1fr) max-content; }
|
||||
.ch > .btn.small { min-width: 0; padding-left: 5px; padding-right: 5px; }
|
||||
.ch > .state { flex: none; }
|
||||
}
|
||||
|
||||
/* settings & login */
|
||||
.settings-layout { display: grid; grid-template-columns: 225px minmax(0, 1fr); gap: 28px; align-items: start; }
|
||||
.settings-menu { position: sticky; top: 76px; border: 1px solid var(--line); border-radius: 12px; padding: 12px; background: var(--card); }
|
||||
.settings-menu-group + .settings-menu-group { margin-top: 18px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.settings-menu-group h2 { margin: 0 8px 7px; font-size: 12px; text-transform: uppercase; letter-spacing: .08em; color: var(--muted); }
|
||||
.settings-menu a { display: block; padding: 8px; border-radius: 7px; text-decoration: none; }
|
||||
.settings-menu a:hover { background: var(--card2); }
|
||||
.settings-menu a[aria-current=page] { background: var(--card2); color: #ffb3bd; box-shadow: inset 3px 0 var(--accent); }
|
||||
.settings-workspace, .settings-content { min-width: 0; }
|
||||
.settings-breadcrumb { margin: 0 0 5px; }
|
||||
.settings-title-row { display: flex; align-items: center; gap: 12px; }
|
||||
.settings-hint { margin: 5px 0 20px; }
|
||||
.settings-content .card { max-width: none; }
|
||||
.catalog-workspace { min-width: 0; margin: 0; padding: 0; border: 0; }
|
||||
.catalog-workspace > .row { margin-bottom: 16px; }
|
||||
.catalog-workspace input, .catalog-workspace select, .catalog-workspace textarea { min-width: 0; max-width: 100%; }
|
||||
.catalog-workspace .card > input, .catalog-workspace .card > textarea { display: block; width: 100%; margin-bottom: 12px; }
|
||||
.catalog-workspace .collection-editor .row { padding: 8px 0; border-bottom: 1px solid var(--line); }
|
||||
.catalog-workspace .collection-editor input[type=number] { width: 80px; }
|
||||
.catalog-workspace .catalog-json { width: 100%; min-height: 280px; font-family: monospace; }
|
||||
@media (max-width: 760px) {
|
||||
.catalog-workspace [data-member] > span { flex-basis: 100%; overflow-wrap: anywhere; }
|
||||
}
|
||||
.settings-tabs { display: flex; flex-wrap: wrap; gap: 8px; margin: 0 0 16px; }
|
||||
.settings-tabs a[aria-current=page] { border-color: var(--accent); color: #ffb3bd; }
|
||||
.scraper-filters { margin: 0 0 18px; }
|
||||
.scraper-filters button[aria-pressed=true] { border-color: var(--accent); color: #ffb3bd; background: color-mix(in srgb, var(--accent) 12%, var(--card)); }
|
||||
.scraper-overview { display: flex; flex-wrap: wrap; gap: 8px; margin: 0 0 18px; }
|
||||
.scraper-overview span { border: 1px solid var(--line); border-radius: 999px; padding: 5px 10px; font-size: 13px; }
|
||||
.scraper-add, .scraper-completed, .scraper-history { border: 1px solid var(--line); border-radius: 9px; padding: 11px 13px; margin: 14px 0; }
|
||||
.scraper-add > summary, .scraper-completed > summary, .scraper-history > summary { cursor: pointer; font-weight: 600; }
|
||||
.scraper-add[open] > summary, .scraper-completed[open] > summary, .scraper-history[open] > summary { margin-bottom: 12px; }
|
||||
.scraper-update-list { margin: 12px 0; border-top: 1px solid var(--line); }
|
||||
.scraper-update-row { display: grid; grid-template-columns: minmax(180px, 1.5fr) minmax(100px, .7fr) repeat(2, minmax(130px, 1fr)); gap: 5px 12px; align-items: center; padding: 11px 0; border-bottom: 1px solid var(--line); font-size: 13px; }
|
||||
.scraper-update-row label { min-width: 0; overflow-wrap: anywhere; }
|
||||
.scraper-update-row .saved { grid-column: 3 / -1; }
|
||||
.scraper-source-card { border: 1px solid var(--line); border-radius: 10px; padding: 12px; margin: 0 0 14px; }
|
||||
.scraper-source-card h3 { margin: 0 0 5px; }
|
||||
.scraper-source-card .source-rate { border-top: 0; }
|
||||
@media (max-width: 760px) {
|
||||
.scraper-update-row { grid-template-columns: minmax(0, 1fr) auto; }
|
||||
.scraper-update-row label { grid-column: 1 / -1; }
|
||||
.scraper-update-row .saved { grid-column: 1 / -1; }
|
||||
}
|
||||
.settings-content .scraper-item .row { flex-shrink: 0; }
|
||||
.settings-menu a:focus-visible, .settings-tabs a:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
||||
@media (max-width: 760px) {
|
||||
.settings-layout { grid-template-columns: minmax(0, 1fr); gap: 18px; }
|
||||
.settings-menu { position: static; padding: 10px; }
|
||||
.settings-menu-group { display: flex; flex-wrap: wrap; gap: 3px; align-items: center; }
|
||||
.settings-menu-group h2 { flex: 0 0 100%; margin: 0 5px 4px; }
|
||||
.settings-menu-group + .settings-menu-group { margin-top: 9px; padding-top: 9px; }
|
||||
.settings-menu a { font-size: 13px; padding: 7px 9px; }
|
||||
.settings-content .scraper-item { flex-wrap: wrap; }
|
||||
}
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 12px; padding: 16px; margin-bottom: 16px; max-width: 720px; }
|
||||
.card h2 { margin: 0 0 12px; font-size: 18px; }
|
||||
.field { margin-bottom: 14px; }
|
||||
@@ -113,6 +263,24 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; }
|
||||
.login { max-width: 360px; margin: 12vh auto 0; }
|
||||
.login input { width: 100%; }
|
||||
.saved { color: #7fd3a0; font-size: 13px; }
|
||||
.reading-data { margin-top: 16px; padding-top: 12px; border-top: 1px solid var(--line); }
|
||||
.reading-data > summary { cursor: pointer; font-weight: 600; }
|
||||
.reading-danger { margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--line); }
|
||||
.reading-danger p { margin: 3px 0 8px; }
|
||||
.scraper-card { max-width: 980px; }
|
||||
.scraper-progress { height: 7px; max-width: 620px; margin: 8px 0 4px; overflow: hidden; border-radius: 5px; background: var(--line); }
|
||||
.scraper-progress i { display: block; height: 100%; background: var(--accent); transition: width .25s; }
|
||||
.scraper-queue { border-top: 1px solid var(--line); }
|
||||
.scraper-item { display: flex; align-items: center; gap: 7px; padding: 8px 0; border-bottom: 1px solid var(--line); }
|
||||
.scraper-item.active { border-left: 3px solid var(--accent); padding-left: 8px; }
|
||||
@media (max-width: 600px) {
|
||||
.scraper-item { flex-wrap: wrap; }
|
||||
.scraper-item > .grow { flex-basis: 100%; width: 100%; }
|
||||
}
|
||||
.source-rate { padding: 9px 0; border-top: 1px solid var(--line); }
|
||||
.source-rate > b { display: block; margin-bottom: 6px; }
|
||||
.rate-row label { display: grid; gap: 3px; color: var(--muted); }
|
||||
.rate-row input { width: 115px; }
|
||||
|
||||
/* ---------------- reader ---------------- */
|
||||
body.reader { position: fixed; top: 0; left: 0; right: 0; bottom: 0; overflow: hidden; background: #0b0b0c; overscroll-behavior: none; }
|
||||
@@ -122,6 +290,7 @@ body.reader { position: fixed; top: 0; left: 0; right: 0; bottom: 0; overflow: h
|
||||
-webkit-user-select: none; user-select: none; -webkit-tap-highlight-color: transparent;
|
||||
overscroll-behavior: contain; scrollbar-width: thin; scrollbar-color: #3a3a40 transparent;
|
||||
}
|
||||
#stage.lock-x { overflow-x: hidden; touch-action: pan-y; overscroll-behavior-x: none; }
|
||||
#stage.pan { cursor: grab; }
|
||||
#stage.drag { cursor: grabbing; }
|
||||
#canvas { position: relative; }
|
||||
@@ -166,10 +335,29 @@ body.nobars #bot { transform: translateY(100%); opacity: 0; pointer-events: none
|
||||
.seg button + button { border-left: 1px solid var(--line); }
|
||||
.seg button.on { background: var(--accent); color: #fff; }
|
||||
.seg button.meh:not(.on) { color: var(--muted); }
|
||||
.zoom-controls button { min-width: 34px; padding-left: 9px; padding-right: 9px; }
|
||||
.zoom-controls #zoom { min-width: 58px; font-variant-numeric: tabular-nums; }
|
||||
.clock-chip { font-variant-numeric: tabular-nums; white-space: nowrap; }
|
||||
.clock-chip.clock-running { border-color: #3d8b62; background: #183426; }
|
||||
.clock-chip.clock-warning { border-color: #c58a27; background: #3b2c14; }
|
||||
.clock-panel {
|
||||
position: fixed; right: max(8px, env(safe-area-inset-right)); bottom: calc(112px + env(safe-area-inset-bottom)); z-index: 25;
|
||||
width: min(360px, calc(100vw - 16px)); padding: 14px; border: 1px solid var(--line); border-radius: 10px;
|
||||
background: rgba(20, 20, 23, .98); color: var(--fg); box-shadow: 0 12px 40px rgba(0, 0, 0, .55);
|
||||
}
|
||||
.clock-head { display: flex; justify-content: space-between; align-items: center; gap: 12px; }
|
||||
.clock-state { color: var(--muted); font-size: 13px; text-align: right; }
|
||||
.clock-details { display: grid; grid-template-columns: auto 1fr; gap: 5px 16px; margin: 12px 0; }
|
||||
.clock-details dt { color: var(--muted); }
|
||||
.clock-details dd { margin: 0; text-align: right; font-variant-numeric: tabular-nums; }
|
||||
#ver { max-width: 170px; }
|
||||
#toast {
|
||||
position: fixed; left: 50%; top: 50%; transform: translate(-50%, -50%); z-index: 30; pointer-events: none;
|
||||
background: rgba(0, 0, 0, .75); color: #fff; padding: 9px 16px; border-radius: 10px; opacity: 0; transition: opacity .25s; text-align: center;
|
||||
}
|
||||
#toast.show { opacity: 1; }
|
||||
@media (max-width: 560px) { .seg button { padding: 7px 10px; } #bot .btn { padding: 6px 10px; } .wide-only { display: none; } }
|
||||
@media (max-width: 700px) { .clock-label { display: none; } }
|
||||
@media (max-width: 560px) {
|
||||
.seg button { padding: 7px 10px; } #bot .btn { padding: 6px 10px; } .wide-only { display: none; }
|
||||
.clock-panel { bottom: calc(124px + env(safe-area-inset-bottom)); }
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Collection · MangaCircus</title><link rel="stylesheet" href="app.css"><script src="common.js" defer></script><script src="collection.js" defer></script></head><body><main><p>Loading…</p></main></body></html>
|
||||
@@ -0,0 +1,20 @@
|
||||
'use strict';
|
||||
(async () => {
|
||||
try {
|
||||
const [me, c] = await Promise.all([api('api/me'), api('api/collection?' + qsa({ id: new URLSearchParams(location.search).get('id') }))]);
|
||||
topNav(me); document.title = c.title + ' · MangaCircus';
|
||||
const main = $('main'); main.textContent = '';
|
||||
main.append(h('section', { class: 'hero' }, c.cover ? h('img', { src: c.cover, alt: '' }) : null,
|
||||
h('div', { class: 'info' }, h('h1', {}, c.title), h('p', { class: 'muted' }, `${c.members.length} manga · ${c.read}/${c.chapters} chapters read · ${c.unread} new`),
|
||||
c.description ? h('p', {}, c.description) : null,
|
||||
c.reading.activeMs ? h('p', {}, `${readingDuration(c.reading.activeMs)} together`) : null,
|
||||
h('div', { class: 'actions' }, h('a', { class: 'btn primary', href: readUrl(c.continue.m, c.continue.c, c.continue.v) }, c.continue.text),
|
||||
me.admin ? h('a', { class: 'btn', href: 'collections.html?' + qsa({ id: c.id }) }, 'Edit collection') : null))));
|
||||
main.append(h('h2', {}, 'Reading order'), h('div', { class: 'collection-members' }, c.members.map(m =>
|
||||
h('a', { class: 'collection-member', href: mangaUrl(m.id) }, m.cover ? h('img', { src: m.cover, alt: '', loading: 'lazy' }) : null,
|
||||
h('div', {}, h('strong', {}, `${m.readingOrder}. ${m.title}`), h('div', { class: 'muted' }, `${m.role.replaceAll('-', ' ')} · ${m.status} · ${m.read}/${m.chapters} read`))))));
|
||||
if (c.relations.length) main.append(h('details', { class: 'card' }, h('summary', {}, 'Relationships & evidence'), c.relations.map(r =>
|
||||
h('p', {}, `${c.members.find(m => m.id === r.from)?.title || r.from} → ${r.type} → ${c.members.find(m => m.id === r.to)?.title || r.to}`,
|
||||
h('span', { class: 'muted' }, ' · ' + r.explanation)))));
|
||||
} catch (e) { showError(e); }
|
||||
})();
|
||||
@@ -0,0 +1,2 @@
|
||||
<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Catalog Curator · MangaCircus</title><link rel="stylesheet" href="app.css"><script src="common.js" defer></script><script src="settings-nav.js" defer></script><script src="collections.js" defer></script></head><body><main><p>Loading…</p></main></body></html>
|
||||
@@ -0,0 +1,126 @@
|
||||
'use strict';
|
||||
// randomUUID is secure-context-only; tailnet HTTP and LAN IPs need this fallback.
|
||||
function collectionId() {
|
||||
if (typeof crypto.randomUUID === 'function') return crypto.randomUUID();
|
||||
const bytes = crypto.getRandomValues(new Uint8Array(16));
|
||||
bytes[6] = (bytes[6] & 15) | 64; bytes[8] = (bytes[8] & 63) | 128;
|
||||
const hex = Array.from(bytes, byte => byte.toString(16).padStart(2, '0')).join('');
|
||||
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
||||
}
|
||||
(async () => {
|
||||
let me, data;
|
||||
try { [me, data] = await Promise.all([api('api/me'), api('api/admin/catalog')]); } catch (e) { return showError(e); }
|
||||
topNav(me);
|
||||
let selected = new URLSearchParams(location.search).get('id');
|
||||
const main = settingsShell(me, 'admin/collections').content;
|
||||
const message = h('p', { role: 'status', 'aria-live': 'polite' });
|
||||
let dirty = false, busy = false, workspace, jsonDraft = null;
|
||||
const notice = (text, error = false) => {
|
||||
message.textContent = text; message.className = error ? 'error' : '';
|
||||
if (error) message.scrollIntoView({ block: 'nearest' });
|
||||
};
|
||||
const changed = () => { dirty = true; notice('Unsaved changes — save the collection to apply them.'); };
|
||||
addEventListener('beforeunload', event => { if (dirty || jsonDraft !== null) { event.preventDefault(); event.returnValue = ''; } });
|
||||
async function request(endpoint, body, success = 'Saved', replaceJson = false) {
|
||||
if (busy) return false;
|
||||
if (jsonDraft !== null && !replaceJson) { notice('Validate & save JSON first, or Reload catalog to discard the JSON draft.', true); return false; }
|
||||
busy = true; workspace.disabled = true; main.setAttribute('aria-busy', 'true'); notice('Saving…');
|
||||
try {
|
||||
data = { ...data, ...await api(endpoint, body) }; dirty = false; jsonDraft = null; busy = false; draw(); notice(success); return true;
|
||||
} catch (e) { notice(e.message + (dirty ? ' Your unsaved changes are still here.' : ''), true); return false; }
|
||||
finally { busy = false; workspace.disabled = false; main.removeAttribute('aria-busy'); }
|
||||
}
|
||||
const persist = () => request('api/admin/catalog', data);
|
||||
const cachedRequest = endpoint => dirty || jsonDraft !== null
|
||||
? notice('Save your changes before importing or researching metadata.', true)
|
||||
: request(endpoint, { m: researchManga, revision: data.revision });
|
||||
let researchManga;
|
||||
const select = (items, value, change, label = 'Catalog selection') => {
|
||||
const input = h('select', { 'aria-label': label, onchange: e => change(e.target.value) }, items.map(item => h('option', { value: typeof item === 'object' ? item.id : item }, typeof item === 'object' ? item.title : item)));
|
||||
input.value = value; return input;
|
||||
};
|
||||
function draw() {
|
||||
main.textContent = '';
|
||||
workspace = h('fieldset', { class: 'catalog-workspace', disabled: busy });
|
||||
main.append(h('p', { class: 'muted' }, 'Collections organize the library. Relationships describe works. Reading order is your choice.'), message, workspace);
|
||||
workspace.append(h('div', { class: 'row' }, select([{ id: '', title: 'Choose collection' }, ...data.collections], selected || '', value => { selected = value; draw(); }, 'Choose collection'),
|
||||
h('button', { class: 'btn', onclick: () => {
|
||||
const unused = data.mangas.find(m => !data.collections.some(c => c.members.some(member => member.mangaId === m.id)));
|
||||
if (!unused) { notice(data.mangas.length ? 'All manga are already grouped. Remove a manga from its current collection before creating another.' : 'Add manga to the library before creating a collection.', true); return; }
|
||||
try {
|
||||
const c = { id: collectionId(), title: 'New collection', description: '', members: [{ mangaId: unused.id, role: 'main', readingOrder: 1 }], coverMode: 'selected', coverMangaId: unused.id, grouped: true };
|
||||
data.collections.push(c); selected = c.id; changed(); draw(); $('input[aria-label="Collection title"]', main).focus();
|
||||
} catch (e) { notice(e.message, true); }
|
||||
} }, 'New collection'),
|
||||
h('button', { class: 'btn', onclick: async () => {
|
||||
if ((dirty || jsonDraft !== null) && !confirm('Discard unsaved catalog changes and reload?')) return;
|
||||
await request('api/admin/catalog', undefined, 'Catalog reloaded', true);
|
||||
} }, 'Reload catalog'), h('button', { class: 'btn', onclick: persist }, 'Save all changes')));
|
||||
const c = data.collections.find(c => c.id === selected);
|
||||
if (c) {
|
||||
const title = h('input', { value: c.title, 'aria-label': 'Collection title', oninput: e => { c.title = e.target.value; changed(); } });
|
||||
const description = h('textarea', { 'aria-label': 'Collection description', oninput: e => { c.description = e.target.value; changed(); } }, c.description);
|
||||
const file = h('input', { type: 'file', accept: 'image/jpeg,image/png,image/webp,image/gif,image/avif,image/bmp', 'aria-label': 'Custom cover', onchange: async e => {
|
||||
const chosen = e.target.files[0]; if (!chosen) return;
|
||||
if (chosen.size > 600000) { notice('Choose an image under 600 KB.', true); return; }
|
||||
// Read first, then serialize save + upload so no other edit can race its revision.
|
||||
let base64;
|
||||
try { base64 = await new Promise((resolve, reject) => {
|
||||
const reader = new FileReader(); reader.onerror = () => reject(new Error('Could not read this image.'));
|
||||
reader.onload = () => resolve(reader.result.split(',')[1]); reader.readAsDataURL(chosen);
|
||||
}); } catch (err) { notice(err.message, true); return; }
|
||||
if (!await persist()) return;
|
||||
await request('api/admin/collection-cover', { id: c.id, revision: data.revision, base64 }, 'Cover saved');
|
||||
} });
|
||||
const members = h('div', { class: 'collection-editor' });
|
||||
for (const member of c.members) members.append(h('div', { class: 'row', 'data-member': member.mangaId },
|
||||
h('span', { class: 'grow' }, data.mangas.find(m => m.id === member.mangaId)?.title || member.mangaId),
|
||||
select(data.roles, member.role, value => { member.role = value; changed(); }, 'Member role'),
|
||||
h('input', { type: 'number', min: 0, value: member.readingOrder, 'aria-label': 'Reading order', oninput: e => { member.readingOrder = +e.target.value; changed(); } }),
|
||||
h('button', { class: 'btn small', disabled: c.members.length === 1, title: c.members.length === 1 ? 'A collection needs a member. Delete the collection to ungroup its last manga.' : 'Remove from this collection', onclick: () => { c.members = c.members.filter(m => m !== member); if (c.coverMangaId === member.mangaId) c.coverMangaId = c.members[0]?.mangaId; changed(); draw(); } }, 'Remove')));
|
||||
const available = data.mangas.filter(m => !data.collections.some(group => group.members.some(member => member.mangaId === m.id)));
|
||||
const add = select([{ id: '', title: available.length ? 'Add a manga…' : 'No ungrouped manga available' }, ...available], '', value => { if (value) { c.members.push({ mangaId: value, role: 'side-story', readingOrder: Math.max(0, ...c.members.map(m => m.readingOrder)) + 1 }); changed(); draw(); } }, 'Add manga to collection');
|
||||
add.disabled = !available.length;
|
||||
workspace.append(h('section', { class: 'card' }, h('h2', {}, 'Collection'), title, description,
|
||||
h('label', { class: 'check' }, h('input', { type: 'checkbox', checked: c.grouped, onchange: e => { c.grouped = e.target.checked; changed(); } }), 'Group on the dashboard'),
|
||||
h('div', { class: 'row' }, h('span', {}, 'Cover'), select(['selected', 'mosaic', ...(c.customCover ? ['custom'] : [])], c.coverMode, value => { c.coverMode = value; changed(); }, 'Cover mode'),
|
||||
select(c.members.map(member => ({ id: member.mangaId, title: data.mangas.find(m => m.id === member.mangaId)?.title || member.mangaId })), c.coverMangaId, value => { c.coverMangaId = value; changed(); }, 'Cover manga'), file),
|
||||
h('p', { class: 'muted small' }, 'Uploading a custom cover saves your changes first. A collection must contain at least one manga.'),
|
||||
h('h3', {}, 'Members & reading order'), members, add,
|
||||
h('div', { class: 'row' }, h('button', { class: 'btn primary', onclick: persist }, 'Save collection'),
|
||||
h('button', { class: 'btn danger', onclick: async () => { if (!confirm('Remove this collection? Its manga and reading progress remain.')) return; data.collections = data.collections.filter(group => group !== c); selected = ''; changed(); if (!await persist()) draw(); } }, 'Delete collection'))));
|
||||
}
|
||||
const from = select(data.mangas, data.mangas[0]?.id, () => {}, 'Relationship source'), to = select(data.mangas, data.mangas[1]?.id, () => {}, 'Relationship target'), type = select(data.types, 'sequel', () => {}, 'Relationship type');
|
||||
workspace.append(h('section', { class: 'card' }, h('h2', {}, 'Relationships'),
|
||||
h('p', { class: 'muted' }, 'Direction: the target is a sequel/prequel/etc. of the source. Suggestions need review; accepting a relationship does not create a collection.'),
|
||||
h('div', { class: 'row' }, from, h('span', {}, '→'), type, h('span', {}, '→'), to,
|
||||
h('button', { class: 'btn', onclick: () => {
|
||||
if (dirty || jsonDraft !== null) return notice('Save your collection changes before editing relationships.', true);
|
||||
if (from.value === to.value) return notice('Choose two different manga for a relationship.', true);
|
||||
if (data.relations.some(r => r.from === from.value && r.to === to.value && r.type === type.value)) return notice('That relationship already exists.', true);
|
||||
data.relations.push({ from: from.value, to: to.value, type: type.value, state: 'confirmed', method: 'manual', evidence: [], explanation: '' }); changed(); persist();
|
||||
} }, 'Add relationship')),
|
||||
h('button', { class: 'btn', onclick: () => cachedRequest('api/admin/catalog/suggest') }, 'Import cached metadata suggestions'),
|
||||
h('div', { class: 'collection-editor' }, data.relations.map(r => h('div', { 'data-relation': JSON.stringify([r.from, r.to, r.type]) },
|
||||
h('p', {}, `${data.mangas.find(m => m.id === r.from)?.title || r.from} → ${r.type} → ${data.mangas.find(m => m.id === r.to)?.title || r.to}`),
|
||||
h('p', { class: 'muted small' }, r.explanation),
|
||||
r.evidence.map(e => h('a', { href: safeHref(e.url), target: '_blank', rel: 'noopener', class: 'btn small' }, e.provider || 'Evidence')),
|
||||
select(['suggested', 'confirmed', 'rejected', 'conflicted'], r.state, value => {
|
||||
if (dirty || jsonDraft !== null) { notice('Save your collection changes before editing relationships.', true); draw(); return; }
|
||||
r.state = value; changed(); persist();
|
||||
}, 'Relationship state'),
|
||||
h('button', { class: 'btn small', onclick: () => {
|
||||
if (dirty || jsonDraft !== null) return notice('Save your collection changes before editing relationships.', true);
|
||||
data.relations = data.relations.filter(item => item !== r); changed(); persist();
|
||||
} }, 'Remove'))))));
|
||||
researchManga = researchManga || data.mangas[0]?.id;
|
||||
const research = select(data.mangas, researchManga, value => researchManga = value, 'Research manga');
|
||||
workspace.append(h('section', { class: 'card' }, h('h2', {}, 'Public metadata research'),
|
||||
h('p', { class: 'muted' }, 'Uses the verified AniList ID and a 30-day cache. Results remain suggestions. An agent can use the JSON export to research conflicts and propose edits.'),
|
||||
h('div', { class: 'row' }, research, h('button', { class: 'btn', onclick: () => cachedRequest('api/admin/catalog/research') }, 'Research relationships'))));
|
||||
const json = h('textarea', { class: 'catalog-json', 'aria-label': 'Catalog JSON', oninput: e => { jsonDraft = e.target.value; notice('Unsaved JSON — use Validate & save JSON to apply it.'); } }, jsonDraft ?? JSON.stringify({ version: 1, revision: data.revision, collections: data.collections, relations: data.relations }, null, 2));
|
||||
workspace.append(h('details', { class: 'card' }, h('summary', {}, 'Advanced · JSON import/export & agent research packet'), json,
|
||||
h('button', { class: 'btn', onclick: async () => { try { const parsed = JSON.parse(json.value); await request('api/admin/catalog', parsed, 'Saved', true); } catch (e) { notice(e.message, true); } } }, 'Validate & save JSON')));
|
||||
}
|
||||
draw();
|
||||
})();
|
||||
@@ -49,9 +49,25 @@ function ago(t) {
|
||||
return new Date(t).toLocaleDateString();
|
||||
}
|
||||
|
||||
function readingDuration(ms, compact = false) {
|
||||
const seconds = Math.max(0, Math.round((+ms || 0) / 1000));
|
||||
if (!seconds) return 'Not timed';
|
||||
if (seconds < 60) return compact ? '<1 min' : 'Under a minute';
|
||||
const minutes = Math.max(1, Math.round(seconds / 60));
|
||||
if (minutes < 60) return `${minutes} min`;
|
||||
const hours = Math.floor(minutes / 60), rest = minutes % 60;
|
||||
return rest ? `${hours} h ${rest} min` : `${hours} h`;
|
||||
}
|
||||
|
||||
const qsa = obj => { const p = new URLSearchParams(); for (const k in obj) if (obj[k] != null && obj[k] !== '') p.set(k, obj[k]); return p.toString(); };
|
||||
const readUrl = (m, c, v) => 'read.html?' + qsa({ m, c, v });
|
||||
const mangaUrl = m => 'manga.html?' + qsa({ m });
|
||||
const libraryUrlForTags = tags => {
|
||||
const p = new URLSearchParams();
|
||||
for (const tag of tags || []) if (tag) p.append('tag', tag);
|
||||
const query = p.toString();
|
||||
return './' + (query ? '?' + query : '');
|
||||
};
|
||||
const safeHref = u => (/^https?:\/\//i.test(u || '') ? u : null);
|
||||
|
||||
// Top navigation with the notification bell.
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Service health · MangaCircus</title><link rel="stylesheet" href="app.css"><script src="common.js" defer></script><script src="settings-nav.js" defer></script><script src="health.js" defer></script></head><body><main><p>Loading…</p></main></body></html>
|
||||
@@ -0,0 +1,34 @@
|
||||
'use strict';
|
||||
(async () => {
|
||||
let me;
|
||||
try { me = await api('api/me'); } catch (e) { return showError(e); }
|
||||
topNav(me);
|
||||
if (!me.admin) return showError(new Error('Administrator access required.'));
|
||||
const main = settingsShell(me, 'admin/health').content;
|
||||
async function load() {
|
||||
try {
|
||||
const [health, scraper] = await Promise.all([api('api/admin/health'), api('api/admin/scraper')]);
|
||||
main.textContent = '';
|
||||
main.append(h('section', { class: 'card' },
|
||||
h('p', {}, `Reader: ${health.reader.ok ? 'healthy' : 'scan error'} · Sentinel: ${health.sentinel.status} · Scraper: ${scraper.status || 'unavailable'}`),
|
||||
h('p', { class: 'muted small' }, health.sentinel.checkedAt ? `Last independent check: ${new Date(health.sentinel.checkedAt).toLocaleString()}` : 'Sentinel has not reported yet.'),
|
||||
h('a', { class: 'btn', href: 'settings.html#admin/scraper' }, 'Queue & hoster settings')));
|
||||
for (const entry of scraper.active || []) main.append(h('section', { class: 'card' }, h('h2', {}, entry.source),
|
||||
h('p', {}, `${entry.current.title} · ${entry.job?.completed || 0}/${entry.job?.total || '?'} chapters`),
|
||||
h('p', { class: 'muted' }, `Checkpoint: ${entry.job?.updated || 'catalog discovery'}`)));
|
||||
const failures = (scraper.queue || []).filter(item => ['error', 'quarantined', 'unavailable', 'retry_wait'].includes(item.status));
|
||||
main.append(h('section', { class: 'card' }, h('h2', {}, 'Jobs needing attention'), failures.length ? failures.map(item => h('div', { class: 'collection-editor' },
|
||||
h('p', {}, `${item.title} · ${item.status} · ${item.failure?.kind || 'unknown'}`), h('p', { class: 'muted' }, item.error || ''),
|
||||
item.nextAttemptAt ? h('p', {}, `Retry scheduled: ${new Date(item.nextAttemptAt).toLocaleString()}`) : null,
|
||||
h('div', { class: 'row' }, h('button', { class: 'btn', onclick: async () => {
|
||||
if (item.failure?.kind === 'access' && !confirm('Has the source access problem been resolved? This manually authorizes one retry.')) return;
|
||||
await api('api/admin/scraper', { command: 'control', action: item.status === 'unavailable' ? 'inspect' : 'retry', id: item.id }); load();
|
||||
} }, item.status === 'unavailable' ? 'Inspect source' : 'Retry'), h('button', { class: 'btn', onclick: async () => { await api('api/admin/scraper', { command: 'control', action: 'skip', id: item.id }); load(); } }, 'Skip job')))) : h('p', { class: 'muted' }, 'No failed jobs.')));
|
||||
const incidents = [...(health.sentinel.incidents || []), ...(scraper.incidents || [])];
|
||||
main.append(h('details', { class: 'card', open: incidents.some(i => i.state === 'open') }, h('summary', {}, 'Incident & recovery history'), incidents.slice(-50).map(i =>
|
||||
h('p', {}, `${i.at} · ${i.state} · ${i.message || i.failure?.message || ''}${i.resolution ? ' · ' + i.resolution : ''}`))));
|
||||
} catch (e) { main.textContent = ''; main.append(h('p', { class: 'error', role: 'alert' }, e.message)); }
|
||||
setTimeout(load, document.hidden ? 30000 : 10000);
|
||||
}
|
||||
load();
|
||||
})();
|
||||
+129
-14
@@ -1,5 +1,5 @@
|
||||
'use strict';
|
||||
/* Library overview: a grid of covers. */
|
||||
/* Library overview: searchable covers with combinable metadata filters. */
|
||||
(async function () {
|
||||
let me, data;
|
||||
try { [me, data] = await Promise.all([api('api/me'), api('api/library')]); } catch (e) { return showError(e); }
|
||||
@@ -14,32 +14,147 @@
|
||||
if (data.scanning) setTimeout(() => location.reload(), 2000);
|
||||
return;
|
||||
}
|
||||
const storyAvailable = !!(data.reading && data.reading.activeMs > 0);
|
||||
|
||||
const filter = h('input', { type: 'search', placeholder: 'Filter…', 'aria-label': 'Filter', value: store.get('filter') || '' });
|
||||
const sort = h('select', { 'aria-label': 'Sort' },
|
||||
h('option', { value: 'title' }, 'A–Z'), h('option', { value: 'updated' }, 'Recently updated'), h('option', { value: 'read' }, 'Recently read'));
|
||||
sort.value = store.get('sort') || 'title';
|
||||
const allTags = [...new Set(data.mangas.flatMap(m => m.tags || []).filter(Boolean))].sort((a, b) => a.localeCompare(b));
|
||||
const allStatuses = [...new Set(data.mangas.map(m => m.status).filter(Boolean))].sort((a, b) => a.localeCompare(b));
|
||||
const requestedTags = new URLSearchParams(location.search).getAll('tag').filter(tag => allTags.includes(tag));
|
||||
let rememberedTags = [];
|
||||
try { rememberedTags = JSON.parse(store.get('tags') || '[]'); } catch (_) { /* ignore old invalid state */ }
|
||||
const selectedTags = new Set((requestedTags.length ? requestedTags : rememberedTags).filter(tag => allTags.includes(tag)));
|
||||
|
||||
const filter = h('input', { type: 'search', placeholder: 'Search titles or authors…', 'aria-label': 'Search library', value: requestedTags.length ? '' : (store.get('filter') || '') });
|
||||
const status = h('select', { 'aria-label': 'Filter by publication status' }, h('option', { value: '' }, 'All statuses'),
|
||||
allStatuses.map(value => h('option', { value }, value)));
|
||||
status.value = !requestedTags.length && allStatuses.includes(store.get('status')) ? store.get('status') : '';
|
||||
const sort = h('select', { 'aria-label': 'Sort library' },
|
||||
h('option', { value: 'title' }, 'Title A–Z'),
|
||||
h('option', { value: 'title-desc' }, 'Title Z–A'),
|
||||
h('option', { value: 'release-desc' }, 'First release · newest'),
|
||||
h('option', { value: 'release-asc' }, 'First release · oldest'),
|
||||
h('option', { value: 'updated' }, 'Recently updated'),
|
||||
h('option', { value: 'read' }, 'Recently read'),
|
||||
storyAvailable ? h('option', { value: 'timed' }, 'Recently timed') : null,
|
||||
storyAvailable ? h('option', { value: 'time' }, 'Time together') : null);
|
||||
const allowedSorts = ['title', 'title-desc', 'release-desc', 'release-asc', 'updated', 'read', ...(storyAvailable ? ['timed', 'time'] : [])];
|
||||
sort.value = allowedSorts.includes(store.get('sort')) ? store.get('sort') : 'title';
|
||||
const resultCount = h('div', { class: 'muted small result-count', 'aria-live': 'polite' });
|
||||
const tagButtons = new Map();
|
||||
const tagBar = h('div', { class: 'tag-filter', 'aria-label': 'Filter by tags' });
|
||||
const clearTags = h('button', { class: 'filter-chip', type: 'button' }, 'All tags');
|
||||
clearTags.onclick = () => { selectedTags.clear(); syncTags(); render(); };
|
||||
tagBar.append(clearTags);
|
||||
for (const tag of allTags) {
|
||||
const button = h('button', { class: 'filter-chip', type: 'button' }, tag);
|
||||
button.onclick = () => { selectedTags.has(tag) ? selectedTags.delete(tag) : selectedTags.add(tag); syncTags(); render(); };
|
||||
tagButtons.set(tag, button);
|
||||
tagBar.append(button);
|
||||
}
|
||||
const grid = h('div', { class: 'grid' });
|
||||
main.append(h('div', { class: 'toolbar' }, filter, sort), grid);
|
||||
const grouping = h('select', { 'aria-label': 'Collection display' },
|
||||
h('option', { value: 'grouped' }, 'Group collections'), h('option', { value: 'expanded' }, 'Expand collections'), h('option', { value: 'individual' }, 'Individual manga'));
|
||||
grouping.value = ['grouped', 'expanded', 'individual'].includes(store.get('grouping')) ? store.get('grouping') : 'grouped';
|
||||
grouping.onchange = render;
|
||||
if (storyAvailable) {
|
||||
const most = data.mangas.find(manga => manga.id === data.reading.mostTimeMangaId);
|
||||
const latest = data.mangas.find(manga => manga.id === data.reading.lastMangaId);
|
||||
main.append(h('section', { class: 'reading-summary' },
|
||||
h('div', { class: 'reading-summary-title' }, h('span', { 'aria-hidden': 'true' }, '⏱'), h('b', {}, 'Your Reading Story')),
|
||||
h('div', { class: 'reading-summary-stats' },
|
||||
h('strong', {}, readingDuration(data.reading.activeMs)),
|
||||
h('span', { class: 'muted' }, `together · ${data.reading.timedChapters} chapter${data.reading.timedChapters === 1 ? '' : 's'} timed`),
|
||||
data.reading.rereads ? h('span', { class: 'story-pill' }, `${data.reading.rereads} reread${data.reading.rereads === 1 ? '' : 's'}`) : null),
|
||||
h('div', { class: 'reading-summary-links' },
|
||||
most ? h('a', { href: mangaUrl(most.id) }, `Most time with ${most.title}`) : null,
|
||||
latest && data.reading.lastChapterId ? h('a', { href: readUrl(latest.id, data.reading.lastChapterId) }, `Continue ${latest.title} →`) : null)));
|
||||
}
|
||||
main.append(
|
||||
h('div', { class: 'library-controls' },
|
||||
h('div', { class: 'toolbar' }, filter, status, sort, grouping,
|
||||
me.admin ? h('a', { class: 'btn', href: 'collections.html' }, 'Manage collections') : null),
|
||||
allTags.length ? h('div', { class: 'filter-group' },
|
||||
h('div', { class: 'filter-heading' }, h('b', {}, 'Tags'), h('span', { class: 'muted small' }, 'Manga must match every selected tag')),
|
||||
tagBar) : null,
|
||||
resultCount),
|
||||
grid);
|
||||
|
||||
function syncTags() {
|
||||
store.set('tags', JSON.stringify([...selectedTags]));
|
||||
const params = new URLSearchParams(location.search);
|
||||
params.delete('tag');
|
||||
for (const tag of selectedTags) params.append('tag', tag);
|
||||
const query = params.toString();
|
||||
history.replaceState(null, '', location.pathname + (query ? '?' + query : ''));
|
||||
}
|
||||
|
||||
function releaseSort(a, b, direction) {
|
||||
const at = +a.firstReleaseAt || 0, bt = +b.firstReleaseAt || 0;
|
||||
if (!at && !bt) return a.title.localeCompare(b.title);
|
||||
if (!at) return 1;
|
||||
if (!bt) return -1;
|
||||
return direction * (at - bt) || a.title.localeCompare(b.title);
|
||||
}
|
||||
|
||||
function render() {
|
||||
store.set('filter', filter.value); store.set('sort', sort.value);
|
||||
store.set('filter', filter.value); store.set('sort', sort.value); store.set('status', status.value); store.set('grouping', grouping.value);
|
||||
const q = filter.value.trim().toLowerCase();
|
||||
const list = data.mangas.filter(m => !q || m.title.toLowerCase().includes(q));
|
||||
const groups = (data.collections || []).filter(c => c.grouped);
|
||||
const membership = new Set(groups.flatMap(c => c.members.map(m => m.id)));
|
||||
const candidates = grouping.value === 'grouped'
|
||||
? [...data.mangas.filter(m => !membership.has(m.id)), ...groups.map(c => ({ ...c, isCollection: true }))] : [...data.mangas];
|
||||
const matches = m => {
|
||||
const haystack = [m.title, ...(m.altTitles || []), ...(m.authors || [])].join(' ').toLowerCase();
|
||||
return (!q || haystack.includes(q)) && (!status.value || m.status === status.value) &&
|
||||
[...selectedTags].every(tag => (m.tags || []).includes(tag));
|
||||
};
|
||||
// Match complete tag combinations on a single member, not unrelated tags spread across a group.
|
||||
const list = candidates.filter(m => m.isCollection ? m.members.some(matches) || (q && m.title.toLowerCase().includes(q) && !status.value && !selectedTags.size) : matches(m));
|
||||
if (sort.value === 'title') list.sort((a, b) => a.title.localeCompare(b.title));
|
||||
if (sort.value === 'title-desc') list.sort((a, b) => b.title.localeCompare(a.title));
|
||||
if (sort.value === 'release-asc') list.sort((a, b) => releaseSort(a, b, 1));
|
||||
if (sort.value === 'release-desc') list.sort((a, b) => releaseSort(a, b, -1));
|
||||
if (sort.value === 'updated') list.sort((a, b) => b.updated - a.updated);
|
||||
if (sort.value === 'read') list.sort((a, b) => b.lastRead - a.lastRead);
|
||||
if (sort.value === 'timed') list.sort((a, b) => (b.reading?.updatedAt || 0) - (a.reading?.updatedAt || 0));
|
||||
if (sort.value === 'time') list.sort((a, b) => (b.reading?.activeMs || 0) - (a.reading?.activeMs || 0));
|
||||
clearTags.classList.toggle('on', !selectedTags.size);
|
||||
clearTags.setAttribute('aria-pressed', String(!selectedTags.size));
|
||||
for (const [tag, button] of tagButtons) {
|
||||
button.classList.toggle('on', selectedTags.has(tag));
|
||||
button.setAttribute('aria-pressed', String(selectedTags.has(tag)));
|
||||
}
|
||||
resultCount.textContent = `${list.length} results · ${data.mangas.length} manga in the library`;
|
||||
grid.textContent = '';
|
||||
for (const m of list) {
|
||||
grid.append(h('a', { class: 'tile', href: mangaUrl(m.id) },
|
||||
h('div', { class: 'cv' }, m.cover ? h('img', { loading: 'lazy', src: m.cover, alt: '' }) : null),
|
||||
const tile = m => {
|
||||
const cover = m.isCollection && m.coverMode === 'mosaic' && m.covers.length
|
||||
? h('div', { class: 'cv collection-mosaic' }, m.covers.map(src => h('img', { loading: 'lazy', src, alt: '' })))
|
||||
: h('div', { class: 'cv' }, m.cover ? h('img', { loading: 'lazy', src: m.cover, alt: '' }) : null);
|
||||
return h('a', { class: 'tile', href: m.isCollection ? 'collection.html?' + qsa({ id: m.id }) : mangaUrl(m.id) },
|
||||
cover, m.isCollection ? h('span', { class: 'collection-count' }, `${m.members.length} manga`) : null,
|
||||
m.unread ? h('span', { class: 'badge' }, m.unread + ' new') : null,
|
||||
h('div', { class: 't' }, m.title),
|
||||
h('div', { class: 's' }, m.read ? `${m.read} / ${m.chapters} read` : `${m.chapters} chapter${m.chapters === 1 ? '' : 's'}`),
|
||||
m.read ? h('div', { class: 'prog' }, h('i', { style: `width:${Math.round((100 * m.read) / m.chapters)}%` })) : null));
|
||||
h('div', { class: 's' }, [m.firstRelease, m.status, m.read ? `${m.read} / ${m.chapters} read` : `${m.chapters} chapters`].filter(Boolean).join(' · ')),
|
||||
m.read ? h('div', { class: 'prog' }, h('i', { style: `width:${Math.round((100 * m.read) / Math.max(1, m.chapters))}%` })) : null);
|
||||
};
|
||||
if (grouping.value === 'expanded') {
|
||||
const visible = new Set(list.map(m => m.id));
|
||||
for (const c of groups) {
|
||||
const members = c.members.filter(m => visible.has(m.id));
|
||||
if (!members.length) continue;
|
||||
grid.append(h('h2', { class: 'collection-heading' }, h('a', { href: 'collection.html?' + qsa({ id: c.id }) }, c.title)));
|
||||
members.forEach(m => { grid.append(tile(m)); visible.delete(m.id); });
|
||||
}
|
||||
if (!list.length) grid.append(h('p', { class: 'muted' }, 'Nothing matches.'));
|
||||
list.filter(m => visible.has(m.id)).forEach(m => grid.append(tile(m)));
|
||||
return;
|
||||
}
|
||||
for (const m of list) {
|
||||
grid.append(tile(m));
|
||||
}
|
||||
if (!list.length) grid.append(h('p', { class: 'muted' }, 'Nothing matches these filters.'));
|
||||
}
|
||||
filter.addEventListener('input', render);
|
||||
status.addEventListener('change', render);
|
||||
sort.addEventListener('change', render);
|
||||
syncTags();
|
||||
render();
|
||||
})();
|
||||
+210
-30
@@ -1,5 +1,5 @@
|
||||
'use strict';
|
||||
/* Manga page: info, follow toggle and the chapter list (books are just separators). */
|
||||
/* Manga overview: complete metadata, volume filters and newest-first chapters. */
|
||||
(async function () {
|
||||
const id = new URLSearchParams(location.search).get('m') || '';
|
||||
let me, M;
|
||||
@@ -11,63 +11,225 @@
|
||||
|
||||
const TYPE = { rtl: 'Manga · right to left', ltr: 'Comic · left to right', ttb: 'Webtoon · vertical strip' };
|
||||
const arr = x => (Array.isArray(x) ? x : x ? [x] : []);
|
||||
const text = value => {
|
||||
if (value == null || value === '') return '';
|
||||
if (Array.isArray(value)) return value.map(text).filter(Boolean).join(', ');
|
||||
if (typeof value === 'object') return JSON.stringify(value);
|
||||
return String(value);
|
||||
};
|
||||
const releaseDate = () => {
|
||||
const raw = meta.firstReleaseDate || meta.releaseDate || meta.startDate;
|
||||
if (!raw) return text(meta.year);
|
||||
if (typeof raw !== 'object') return text(raw);
|
||||
return [raw.year, raw.month && String(raw.month).padStart(2, '0'), raw.day && String(raw.day).padStart(2, '0')].filter(Boolean).join('-');
|
||||
};
|
||||
const follow = h('button', { class: 'btn' }, '');
|
||||
const drawFollow = () => { follow.textContent = M.following ? '🔔 Notifying you' : 'Notify me about new chapters'; follow.classList.toggle('on', M.following); };
|
||||
follow.onclick = async () => { M.following = (await api('api/follow', { m: M.id, on: !M.following })).following; drawFollow(); };
|
||||
drawFollow();
|
||||
|
||||
const sources = arr(meta.sources).filter(s => s && s.name);
|
||||
const sources = arr(meta.sources).filter(source => source && (source.name || source.url));
|
||||
const tags = arr(meta.tags).filter(Boolean);
|
||||
const books = {};
|
||||
for (const book of M.books) books[book.key] = book;
|
||||
const selectedBooks = new Set();
|
||||
main.append(h('section', { class: 'hero' },
|
||||
M.cover ? h('img', { src: M.cover, alt: '' }) : null,
|
||||
h('div', { class: 'info' },
|
||||
h('h1', {}, M.title),
|
||||
arr(meta.altTitles).length ? h('div', { class: 'muted' }, arr(meta.altTitles).join(' · ')) : null,
|
||||
h('div', { class: 'muted' }, [arr(meta.authors).join(', '), meta.status, meta.year, TYPE[M.direction]].filter(Boolean).join(' · ')),
|
||||
h('div', { class: 'chips' }, arr(meta.tags).map(t => h('span', { class: 'chip' }, t))),
|
||||
h('div', { class: 'muted' }, [arr(meta.authors).join(', '), meta.status, releaseDate(), TYPE[M.direction]].filter(Boolean).join(' · ')),
|
||||
h('div', { class: 'chips' }, tags.map(tag => h('a', { class: 'chip chip-link', href: libraryUrlForTags([tag]), title: `Show manga tagged ${tag}` }, tag))),
|
||||
meta.description ? h('p', { class: 'desc' }, meta.description) : null,
|
||||
sources.length ? h('div', { class: 'sources small' }, h('span', { class: 'muted' }, 'Sources: '),
|
||||
sources.map(s => (safeHref(s.url) ? h('a', { href: safeHref(s.url), target: '_blank', rel: 'noopener noreferrer' }, s.name) : h('span', {}, s.name + ' ')))) : null,
|
||||
h('div', { class: 'actions' },
|
||||
M.continue ? h('a', { class: 'btn primary', href: readUrl(M.id, M.continue.c, M.continue.v) }, M.continue.text) : null,
|
||||
follow))));
|
||||
|
||||
const books = {};
|
||||
for (const b of M.books) books[b.key] = b;
|
||||
let storyRhythm = null;
|
||||
if (M.collections?.length || M.relations?.length) main.append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Collections & related manga'),
|
||||
h('div', { class: 'chips' }, (M.collections || []).map(c => h('a', { class: 'btn', href: 'collection.html?' + qsa({ id: c.id }) }, c.title))),
|
||||
(M.relations || []).map(r => h('p', {}, h('a', { href: mangaUrl(r.otherId) }, r.otherTitle),
|
||||
` · ${r.from === M.id ? 'target is ' : 'this manga is '}${r.type.replaceAll('-', ' ')}`,
|
||||
h('details', {}, h('summary', {}, 'Evidence'), h('p', { class: 'muted' }, r.explanation),
|
||||
r.evidence.map(e => h('a', { class: 'btn small', href: safeHref(e.url), target: '_blank', rel: 'noopener' }, e.provider || 'Source')))))));
|
||||
if (M.reading && M.reading.activeMs > 0) {
|
||||
const lastChapter = M.chapters.find(chapter => chapter.id === M.reading.lastChapterId);
|
||||
storyRhythm = h('div', { class: 'story-rhythm', role: 'list', 'aria-label': 'Completed chapter reading times' });
|
||||
const openKey = `reading-story-open.${me.name}`;
|
||||
const story = h('details', { class: 'reading-story card', open: store.get(openKey) !== '0' },
|
||||
h('summary', { class: 'reading-story-summary' },
|
||||
h('span', {}, h('span', { 'aria-hidden': 'true' }, '⏱ '), h('b', {}, 'Your Reading Story')),
|
||||
h('span', { class: 'muted small' }, readingDuration(M.reading.activeMs) + ' together')),
|
||||
h('div', { class: 'story-stats' },
|
||||
storyStat(readingDuration(M.reading.activeMs), 'time together'),
|
||||
storyStat(M.reading.timedChapters, `chapter${M.reading.timedChapters === 1 ? '' : 's'} timed`),
|
||||
storyStat(M.reading.typicalChapterMs ? readingDuration(M.reading.typicalChapterMs) : '—', 'typical chapter'),
|
||||
storyStat(M.reading.rereads, `reread${M.reading.rereads === 1 ? '' : 's'}`)),
|
||||
lastChapter ? h('p', { class: 'story-last muted' }, 'Last timed ',
|
||||
h('a', { href: readUrl(M.id, lastChapter.id) }, `${lastChapter.label}${lastChapter.title ? ' · ' + lastChapter.title : ''}`),
|
||||
` · ${ago(M.reading.updatedAt)}`) : null,
|
||||
h('div', { class: 'story-rhythm-head' }, h('b', {}, 'Reading rhythm'), h('span', { class: 'muted small' }, 'Completed chapters · active time')),
|
||||
storyRhythm,
|
||||
h('div', { class: 'story-actions' },
|
||||
h('span', { class: 'muted small' }, 'Only you can see these measurements.'),
|
||||
h('button', { class: 'btn small', onclick: async () => {
|
||||
if (!confirm(`Forget all reading time recorded for ${M.title}? Reading progress is kept.`)) return;
|
||||
await api('api/reading/delete', { m: M.id });
|
||||
location.reload();
|
||||
} }, 'Forget this manga’s time')));
|
||||
story.addEventListener('toggle', () => store.set(openKey, story.open ? '1' : '0'));
|
||||
main.append(story);
|
||||
drawStoryRhythm();
|
||||
}
|
||||
|
||||
function storyStat(value, label) {
|
||||
return h('div', { class: 'story-stat' }, h('strong', {}, value), h('span', { class: 'muted small' }, label));
|
||||
}
|
||||
|
||||
function drawStoryRhythm() {
|
||||
if (!storyRhythm) return;
|
||||
const chapters = M.chapters.filter(chapter => !selectedBooks.size || selectedBooks.has(chapter.book));
|
||||
const completed = chapters.map(chapter => chapter.reading?.lastCompleteMs || 0).filter(Boolean);
|
||||
const scale = Math.max(1, ...completed.map(value => Math.log1p(value)));
|
||||
storyRhythm.textContent = '';
|
||||
for (const chapter of chapters) {
|
||||
const value = chapter.reading?.lastCompleteMs || 0;
|
||||
const label = `${chapter.label}${chapter.title ? ' · ' + chapter.title : ''}`;
|
||||
if (!value) {
|
||||
storyRhythm.append(h('span', { class: 'rhythm-bar empty', role: 'listitem', title: `${label} · Not timed`, 'aria-label': `${label}, not timed` }));
|
||||
continue;
|
||||
}
|
||||
const height = Math.max(12, Math.round(12 + 48 * Math.log1p(value) / scale));
|
||||
const passes = chapter.reading.completedPasses;
|
||||
const title = `${label} · ${readingDuration(value)}${passes > 1 ? ` · ${passes} timed reads · ${readingDuration(chapter.reading.totalActiveMs)} total` : ''}`;
|
||||
storyRhythm.append(h('a', {
|
||||
class: 'rhythm-bar', role: 'listitem', href: readUrl(M.id, chapter.id), title,
|
||||
'aria-label': title, style: `height:${height}px`,
|
||||
}));
|
||||
}
|
||||
if (!chapters.length) storyRhythm.append(h('span', { class: 'muted small' }, 'No chapters in the selected volumes.'));
|
||||
}
|
||||
|
||||
const metadata = h('dl', { class: 'meta-grid' });
|
||||
function addMeta(label, value) {
|
||||
if (value == null || value === '' || (Array.isArray(value) && !value.length)) return;
|
||||
metadata.append(h('dt', {}, label), h('dd', {}, value instanceof Node ? value : text(value)));
|
||||
}
|
||||
addMeta('Title', meta.title || M.title);
|
||||
addMeta('Alternative titles', arr(meta.altTitles));
|
||||
addMeta('Authors', arr(meta.authors));
|
||||
addMeta('Status', meta.status);
|
||||
addMeta('First release', releaseDate());
|
||||
addMeta('Format', meta['x-anilist'] && meta['x-anilist'].format);
|
||||
addMeta('Reading style', TYPE[M.direction] || M.direction);
|
||||
addMeta('Volumes', M.books.length);
|
||||
addMeta('Chapters', M.chapters.length);
|
||||
addMeta('Library ID', M.id);
|
||||
addMeta('Tags', h('span', { class: 'meta-tags' }, tags.map(tag => h('a', { class: 'chip chip-link', href: libraryUrlForTags([tag]) }, tag))));
|
||||
if (sources.length) addMeta('Content sources', h('span', { class: 'meta-links' }, sources.map(source => {
|
||||
const label = source.name || source.url;
|
||||
return safeHref(source.url) ? h('a', { href: safeHref(source.url), target: '_blank', rel: 'noopener noreferrer' }, label) : h('span', {}, label);
|
||||
})));
|
||||
|
||||
const anilist = meta['x-anilist'] || {};
|
||||
if (anilist.url || anilist.id) addMeta('AniList', safeHref(anilist.url)
|
||||
? h('a', { href: safeHref(anilist.url), target: '_blank', rel: 'noopener noreferrer' }, anilist.id ? `Title ${anilist.id}` : 'Open title')
|
||||
: anilist.id);
|
||||
if (anilist.idMal) addMeta('MyAnimeList', h('a', { href: `https://myanimelist.net/manga/${encodeURIComponent(anilist.idMal)}`, target: '_blank', rel: 'noopener noreferrer' }, `Title ${anilist.idMal}`));
|
||||
if (anilist.coverUrl) addMeta('Metadata cover', safeHref(anilist.coverUrl)
|
||||
? h('a', { href: safeHref(anilist.coverUrl), target: '_blank', rel: 'noopener noreferrer' }, 'Open source image') : anilist.coverUrl);
|
||||
|
||||
const cover = meta['x-cover'] || {};
|
||||
addMeta('Cover provider', [cover.source, cover.label].filter(Boolean).join(' · '));
|
||||
if (cover.url) addMeta('Cover source', safeHref(cover.url)
|
||||
? h('a', { href: safeHref(cover.url), target: '_blank', rel: 'noopener noreferrer' }, cover.url) : cover.url);
|
||||
const scraper = meta['x-scraper'] || {};
|
||||
addMeta('Auto update', typeof scraper.autoUpdate === 'boolean' ? (scraper.autoUpdate ? 'Enabled' : 'Disabled') : '');
|
||||
addMeta('Scraper strategy', scraper.strategy);
|
||||
addMeta('Source identifiers', scraper.ids);
|
||||
|
||||
const handled = new Set(['title', 'altTitles', 'authors', 'status', 'year', 'firstReleaseDate', 'releaseDate', 'startDate', 'tags', 'description', 'direction', 'sources', 'x-anilist', 'x-cover', 'x-scraper']);
|
||||
for (const [key, value] of Object.entries(meta)) if (!handled.has(key)) addMeta(key, value);
|
||||
const rawMeta = h('details', { class: 'raw-meta' },
|
||||
h('summary', {}, 'Raw metadata'),
|
||||
h('pre', {}, JSON.stringify(meta, null, 2)));
|
||||
const metadataOpen = !window.matchMedia('(max-width: 640px)').matches;
|
||||
main.append(h('details', { class: 'metadata card', open: metadataOpen },
|
||||
h('summary', { class: 'metadata-summary' }, h('h2', {}, 'Metadata')),
|
||||
metadata, rawMeta));
|
||||
|
||||
const volumeButtons = new Map();
|
||||
const volumeBar = h('div', { class: 'volume-filter', 'aria-label': 'Filter chapters by volume' });
|
||||
const allVolumes = h('button', { class: 'filter-chip', type: 'button' }, 'All volumes');
|
||||
allVolumes.onclick = () => { selectedBooks.clear(); drawVolumeFilter(); drawChapters(); drawStoryRhythm(); };
|
||||
volumeBar.append(allVolumes);
|
||||
for (const book of M.books) {
|
||||
const label = book.label + (book.title ? ' · ' + book.title : '');
|
||||
const button = h('button', { class: 'filter-chip', type: 'button', title: label }, label);
|
||||
button.onclick = () => { selectedBooks.has(book.key) ? selectedBooks.delete(book.key) : selectedBooks.add(book.key); drawVolumeFilter(); drawChapters(); drawStoryRhythm(); };
|
||||
volumeButtons.set(book.key, button);
|
||||
volumeBar.append(button);
|
||||
}
|
||||
const list = h('section', { class: 'chapters' });
|
||||
main.append(list);
|
||||
const empty = h('p', { class: 'muted chapter-empty', hidden: true }, 'No chapters match these volumes.');
|
||||
main.append(h('section', { class: 'chapter-section' },
|
||||
h('div', { class: 'chapter-heading' }, h('h2', {}, 'Chapters'), h('span', { class: 'muted small' }, 'Newest first')),
|
||||
volumeBar,
|
||||
h('div', { class: 'chapter-list-wrap' }, list), empty));
|
||||
|
||||
function drawVolumeFilter() {
|
||||
allVolumes.classList.toggle('on', !selectedBooks.size);
|
||||
allVolumes.setAttribute('aria-pressed', String(!selectedBooks.size));
|
||||
for (const [key, button] of volumeButtons) {
|
||||
button.classList.toggle('on', selectedBooks.has(key));
|
||||
button.setAttribute('aria-pressed', String(selectedBooks.has(key)));
|
||||
}
|
||||
}
|
||||
|
||||
function drawChapters() {
|
||||
list.textContent = '';
|
||||
const chapters = [...M.chapters].reverse().filter(chapter => !selectedBooks.size || selectedBooks.has(chapter.book));
|
||||
empty.hidden = !!chapters.length;
|
||||
let lastBook = null;
|
||||
for (const ch of M.chapters) {
|
||||
if (ch.book !== lastBook && M.books.length > 1) {
|
||||
const b = books[ch.book];
|
||||
list.append(h('div', { class: 'book-sep' }, b.label + (b.title ? ' · ' + b.title : '')));
|
||||
for (const chapter of chapters) {
|
||||
if (chapter.book !== lastBook && M.books.length > 1) {
|
||||
const book = books[chapter.book];
|
||||
list.append(h('div', { class: 'book-sep' }, book.label + (book.title ? ' · ' + book.title : '')));
|
||||
}
|
||||
lastBook = chapter.book;
|
||||
list.append(chapterRow(chapter));
|
||||
}
|
||||
lastBook = ch.book;
|
||||
list.append(chapterRow(ch));
|
||||
}
|
||||
|
||||
function chapterRow(ch) {
|
||||
const wrap = h('div', {});
|
||||
const wrap = h('div', { class: 'chapter-entry' });
|
||||
const panel = h('div', { class: 'versions', hidden: true });
|
||||
function draw() {
|
||||
const p = ch.progress, done = p && p.done;
|
||||
const state = h('button', { class: 'state' + (done ? ' done' : ''), title: done ? 'Mark as unread' : 'Mark as read' },
|
||||
done ? '✓ read' : p ? `page ${p.p + 1} / ${p.n}` : `${ch.versions[0].pages} p.`);
|
||||
done ? '✓ read' : p ? `page ${p.p + 1} / ${p.n}` : 'unread');
|
||||
state.onclick = async () => {
|
||||
if (done) { await api('api/unread', { m: M.id, c: ch.id }); ch.progress = null; }
|
||||
else {
|
||||
const v = ch.versions[0];
|
||||
await api('api/progress', { m: M.id, c: ch.id, v: v.id, p: v.pages - 1, n: v.pages, done: true });
|
||||
ch.progress = { v: v.id, p: v.pages - 1, n: v.pages, done: true };
|
||||
const version = ch.versions[0];
|
||||
await api('api/progress', { m: M.id, c: ch.id, v: version.id, p: version.pages - 1, n: version.pages, done: true });
|
||||
ch.progress = { v: version.id, p: version.pages - 1, n: version.pages, done: true };
|
||||
}
|
||||
draw();
|
||||
};
|
||||
const verBtn = ch.versions.length > 1
|
||||
? h('button', { class: 'btn small', onclick: () => { panel.hidden = !panel.hidden; } }, ch.versions.length + ' versions')
|
||||
: null;
|
||||
const pages = ch.versions[0] ? ch.versions[0].pages : 0;
|
||||
const timing = chapterTiming(ch);
|
||||
wrap.textContent = '';
|
||||
wrap.append(h('div', { class: 'ch' + (done ? ' read' : '') },
|
||||
h('a', { class: 'ch-main', href: readUrl(M.id, ch.id) },
|
||||
h('span', { class: 'ch-label' }, ch.label), h('span', { class: 'ch-title' }, ch.title || ''),
|
||||
h('a', { class: 'ch-main' + (timing ? ' has-time' : ''), href: readUrl(M.id, ch.id) },
|
||||
h('span', { class: 'ch-label' }, ch.label),
|
||||
h('span', { class: 'ch-pages' }, `${pages} p.`),
|
||||
timing ? h('span', { class: 'ch-time', title: timing.title, 'aria-label': timing.title }, timing.text) : null,
|
||||
h('span', { class: 'ch-title', title: ch.title || '' }, ch.title || 'Untitled'),
|
||||
ch.extra ? h('span', { class: 'tag' }, 'extra') : null),
|
||||
verBtn, state), panel);
|
||||
drawVersions();
|
||||
@@ -75,21 +237,39 @@
|
||||
function drawVersions() {
|
||||
panel.textContent = '';
|
||||
panel.append(h('div', { class: 'muted small', style: 'padding-top:6px' }, 'The best rated version opens by default. Vote to change that for everyone.'));
|
||||
ch.versions.forEach((v, i) => {
|
||||
const vote = async value => { ch.versions = (await api('api/vote', { m: M.id, c: ch.id, v: v.id, value: v.my === value ? 0 : value })).versions; drawVersions(); };
|
||||
ch.versions.forEach((version, i) => {
|
||||
const vote = async value => { ch.versions = (await api('api/vote', { m: M.id, c: ch.id, v: version.id, value: version.my === value ? 0 : value })).versions; drawVersions(); };
|
||||
panel.append(h('div', { class: 'ver' },
|
||||
h('div', { class: 'src' },
|
||||
safeHref(v.sourceUrl) ? h('a', { href: safeHref(v.sourceUrl), target: '_blank', rel: 'noopener noreferrer' }, v.source || v.id) : h('span', {}, v.source || v.id),
|
||||
safeHref(version.sourceUrl) ? h('a', { href: safeHref(version.sourceUrl), target: '_blank', rel: 'noopener noreferrer' }, version.source || version.id) : h('span', {}, version.source || version.id),
|
||||
i === 0 ? h('span', { class: 'tag', style: 'margin-left:8px' }, 'default') : null,
|
||||
h('div', { class: 'muted small' }, `${v.pages} pages · added ${new Date(v.added).toLocaleDateString()}`)),
|
||||
h('div', { class: 'muted small' }, `${version.pages} pages · added ${new Date(version.added).toLocaleDateString()}`)),
|
||||
h('span', { class: 'vote' },
|
||||
h('button', { class: v.my > 0 ? 'on' : '', title: 'Better', onclick: () => vote(1) }, '▲'),
|
||||
h('b', {}, v.score),
|
||||
h('button', { class: v.my < 0 ? 'on' : '', title: 'Worse', onclick: () => vote(-1) }, '▼')),
|
||||
h('a', { class: 'btn small', href: readUrl(M.id, ch.id, v.id) }, 'Read')));
|
||||
h('button', { class: version.my > 0 ? 'on' : '', title: 'Better', onclick: () => vote(1) }, '▲'),
|
||||
h('b', {}, version.score),
|
||||
h('button', { class: version.my < 0 ? 'on' : '', title: 'Worse', onclick: () => vote(-1) }, '▼')),
|
||||
h('a', { class: 'btn small', href: readUrl(M.id, ch.id, version.id) }, 'Read')));
|
||||
});
|
||||
}
|
||||
draw();
|
||||
return wrap;
|
||||
}
|
||||
|
||||
function chapterTiming(chapter) {
|
||||
if (!M.reading) return null;
|
||||
const value = chapter.reading;
|
||||
if (!value) return { text: 'Not timed', title: 'No reading time recorded' };
|
||||
if (value.completedPasses) {
|
||||
const text = readingDuration(value.lastCompleteMs, true) + (value.completedPasses > 1 ? ` ↻${value.completedPasses}` : '');
|
||||
return {
|
||||
text,
|
||||
title: `Last timed read ${readingDuration(value.lastCompleteMs)}; ${value.completedPasses} timed read${value.completedPasses === 1 ? '' : 's'}; ${readingDuration(value.totalActiveMs)} total`,
|
||||
};
|
||||
}
|
||||
const active = value.currentPassMs || value.totalActiveMs;
|
||||
return { text: `◷ ${readingDuration(active, true)}`, title: `${readingDuration(active)} active; current pass in progress` };
|
||||
}
|
||||
|
||||
drawVolumeFilter();
|
||||
drawChapters();
|
||||
})();
|
||||
+9
-1
@@ -12,6 +12,8 @@
|
||||
<link rel="manifest" href="manifest.webmanifest">
|
||||
<meta name="theme-color" content="#0b0b0c">
|
||||
<script src="common.js" defer></script>
|
||||
<script src="reading-clock-core.js" defer></script>
|
||||
<script src="reading-clock.js" defer></script>
|
||||
<script src="reader.js" defer></script>
|
||||
</head>
|
||||
<body class="reader">
|
||||
@@ -37,11 +39,17 @@
|
||||
<button data-mode="book" title="Two pages side by side, like an open book">Book</button>
|
||||
</div>
|
||||
<button id="shift" class="btn" title="Page pairs look wrong? Shift them by one page">⇄<span class="wide-only"> Shift</span></button>
|
||||
<button id="zoom" class="btn" title="Page zoom (Ctrl + scroll or pinch). Click to fit the screen again.">100%</button>
|
||||
<div class="seg zoom-controls" role="group" aria-label="Page zoom">
|
||||
<button id="zoomout" title="Zoom out 5%" aria-label="Zoom out 5%">−</button>
|
||||
<button id="zoom" title="Current page zoom. Click to fit the screen again.">100%</button>
|
||||
<button id="zoomin" title="Zoom in 5%" aria-label="Zoom in 5%">+</button>
|
||||
</div>
|
||||
<button id="readclock" class="btn clock-chip" hidden aria-label="Active Reading Clock"><span class="clock-icon">⏸</span> <span class="clock-time">00:00</span><span class="clock-label"></span></button>
|
||||
<select id="ver" title="Version of this chapter" aria-label="Version"></select>
|
||||
<a id="nextch" class="btn" title="Next chapter"><span class="wide-only">Next </span>›</a>
|
||||
</div>
|
||||
</div>
|
||||
<div id="clockpanel" class="clock-panel" hidden></div>
|
||||
<div id="toast"></div>
|
||||
</body>
|
||||
</html>
|
||||
+81
-13
@@ -16,7 +16,7 @@
|
||||
catch (e) { canvas.textContent = ''; canvas.append(h('p', { class: 'loading' }, e.message + ' · ', h('a', { href: './' }, 'Library'))); return; }
|
||||
|
||||
const S = me.settings, rtl = D.direction === 'rtl', ttb = D.direction === 'ttb';
|
||||
const GAP = ttb ? 0 : 8, END_H = 280;
|
||||
const GAP = ttb || S.flowPageGap === false ? 0 : 8, END_H = 280;
|
||||
document.title = `${D.chapter.label} · ${D.manga.title}`;
|
||||
|
||||
// ---------- pages ----------
|
||||
@@ -41,6 +41,7 @@
|
||||
|
||||
let cur = 0, atEnd = false, done = !!(D.progress && D.progress.done), bars = true, barsAt = 0;
|
||||
let spreads = [], spreadIdx = 0, shown = [], els = [], boxes = [], endEl = null, lastW = 0, lastH = 0;
|
||||
let lockFlowX = false, readingClock = null;
|
||||
const shift = new Set(D.progress && D.progress.v === D.version ? D.progress.shift : []);
|
||||
const zkey = () => (mode === 'flow' && ttb ? 'strip' : mode);
|
||||
const zoom = () => S.zoom[zkey()] || 1;
|
||||
@@ -112,8 +113,9 @@
|
||||
Object.assign(p, { unknown: false, w, a: w / hh, wide: p.wide || w > hh });
|
||||
relayout();
|
||||
}
|
||||
if (readingClock) readingClock.viewChanged();
|
||||
};
|
||||
img.onerror = state;
|
||||
img.onerror = () => { state(); if (readingClock) readingClock.viewChanged(); };
|
||||
state();
|
||||
}
|
||||
function setSrc(el, url) { el.firstChild.src = url; watch(el); }
|
||||
@@ -162,6 +164,7 @@
|
||||
}))).then(() => { if (token === buildToken) preloadAround(); });
|
||||
}
|
||||
layout();
|
||||
if (readingClock) readingClock.viewChanged();
|
||||
}
|
||||
let buildToken = 0;
|
||||
|
||||
@@ -198,7 +201,11 @@
|
||||
canvas.style.width = cw + 'px';
|
||||
canvas.style.height = ch + 'px';
|
||||
els.forEach((el, j) => place(el, boxes[j]));
|
||||
stage.classList.toggle('pan', cw > vw + 1 || (mode === 'flow' ? z > 1.01 : ch > vh + 1));
|
||||
const overflowsX = cw > vw + 1;
|
||||
lockFlowX = mode === 'flow' && !overflowsX;
|
||||
stage.classList.toggle('pan', overflowsX || (mode !== 'flow' && ch > vh + 1));
|
||||
stage.classList.toggle('lock-x', lockFlowX);
|
||||
if (lockFlowX) stage.scrollLeft = 0;
|
||||
lastW = vw; lastH = vh;
|
||||
}
|
||||
|
||||
@@ -230,17 +237,18 @@
|
||||
function restore(a, cx, cy) {
|
||||
if (!a || !boxes.length) return;
|
||||
const b = a.k < 0 ? union() : boxes[a.k];
|
||||
stage.scrollLeft = b.x + a.fx * b.w - cx;
|
||||
stage.scrollLeft = lockFlowX ? 0 : b.x + a.fx * b.w - cx;
|
||||
stage.scrollTop = b.y + a.fy * b.h - cy;
|
||||
barsAt = stage.scrollTop;
|
||||
}
|
||||
function relayout() { const a = anchor(lastW / 2, lastH / 2); layout(); restore(a, lastW / 2, lastH / 2); }
|
||||
|
||||
// ---------- zoom ----------
|
||||
const MIN_ZOOM = 0.2, MAX_ZOOM = 6, ZOOM_STEP = 0.05;
|
||||
let zoomTimer = 0;
|
||||
function setZoom(z, cx = stage.clientWidth / 2, cy = stage.clientHeight / 2) {
|
||||
z = Math.min(6, Math.max(0.2, z));
|
||||
if (Math.abs(z - 1) < 0.03) z = 1; // easy to get back to "fit"
|
||||
z = Math.min(MAX_ZOOM, Math.max(MIN_ZOOM, z));
|
||||
if (Math.abs(z - 1) < 0.01) z = 1; // easy to get back to "fit" without swallowing fine adjustments
|
||||
if (Math.abs(z - zoom()) < 0.001) return;
|
||||
const a = anchor(cx, cy);
|
||||
S.zoom[zkey()] = z;
|
||||
@@ -252,11 +260,39 @@
|
||||
zoomTimer = setTimeout(() => api('api/settings', { zoom: S.zoom }).catch(() => {}), 800);
|
||||
}
|
||||
|
||||
// Pages exposed enough to count as something the reader could actually inspect.
|
||||
// The clock handles decoding, duration and plausibility; this keeps layout knowledge here.
|
||||
function readingView() {
|
||||
const viewport = stage.getBoundingClientRect(), viewportArea = viewport.width * viewport.height, pages = [];
|
||||
for (let j = 0; j < els.length; j++) {
|
||||
const el = els[j], rect = el.getBoundingClientRect();
|
||||
const w = Math.max(0, Math.min(rect.right, viewport.right) - Math.max(rect.left, viewport.left));
|
||||
const hh = Math.max(0, Math.min(rect.bottom, viewport.bottom) - Math.max(rect.top, viewport.top));
|
||||
const visibleArea = w * hh, pageArea = Math.max(1, rect.width * rect.height);
|
||||
if (visibleArea < 0.10 * Math.min(pageArea, viewportArea)) continue;
|
||||
const img = el.firstChild, page = mode === 'flow' ? list[j] : list[shown[j]];
|
||||
pages.push({
|
||||
index: page.i,
|
||||
loaded: !!(img.complete && img.naturalWidth),
|
||||
failed: !!(img.complete && img.getAttribute('src') && !img.naturalWidth),
|
||||
});
|
||||
}
|
||||
const last = boxes[boxes.length - 1];
|
||||
const flowEnd = mode === 'flow' && last && !pages.length && stage.scrollTop + stage.clientHeight >= last.y + last.h + 32;
|
||||
return { pages, atEnd: mode === 'flow' ? !!flowEnd : atEnd };
|
||||
}
|
||||
|
||||
function stepZoom(direction) {
|
||||
const units = zoom() / ZOOM_STEP;
|
||||
const stepped = (direction > 0 ? Math.floor(units + 1e-6) + 1 : Math.ceil(units - 1e-6) - 1) * ZOOM_STEP;
|
||||
setZoom(Math.round(stepped * 100) / 100);
|
||||
}
|
||||
|
||||
// Ctrl + wheel (and touchpad pinch, which browsers report the same way) zooms the pages, not the page controls.
|
||||
stage.addEventListener('wheel', e => {
|
||||
if (!e.ctrlKey) return;
|
||||
e.preventDefault();
|
||||
setZoom(zoom() * Math.exp(-e.deltaY * (e.deltaMode === 1 ? 0.05 : 0.0025)), e.clientX, e.clientY);
|
||||
setZoom(zoom() * Math.exp(-e.deltaY * (e.deltaMode === 1 ? 0.015 : 0.0005)), e.clientX, e.clientY);
|
||||
}, { passive: false });
|
||||
|
||||
// Touch: two fingers zoom, a quick horizontal swipe turns the page (page/book mode, when not zoomed in).
|
||||
@@ -266,10 +302,18 @@
|
||||
stage.addEventListener('touchstart', e => {
|
||||
const t = e.touches;
|
||||
if (t.length === 2) { pinch = { d: dist(t), z: zoom() }; swipe = null; }
|
||||
else if (t.length === 1) swipe = { x: t[0].clientX, y: t[0].clientY, t: Date.now() };
|
||||
else if (t.length === 1) swipe = { x: t[0].clientX, y: t[0].clientY, top: stage.scrollTop, t: Date.now() };
|
||||
}, { passive: true });
|
||||
stage.addEventListener('touchmove', e => {
|
||||
if (!pinch || e.touches.length !== 2) return;
|
||||
if (!pinch || e.touches.length !== 2) {
|
||||
const t = e.touches[0], s = swipe;
|
||||
if (t && s && lockFlowX && Math.abs(t.clientX - s.x) > Math.abs(t.clientY - s.y)) {
|
||||
e.preventDefault();
|
||||
stage.scrollLeft = 0;
|
||||
stage.scrollTop = s.top - (t.clientY - s.y);
|
||||
}
|
||||
return;
|
||||
}
|
||||
e.preventDefault();
|
||||
const t = e.touches;
|
||||
pinchNext = [pinch.z * dist(t) / pinch.d, (t[0].clientX + t[1].clientX) / 2, (t[0].clientY + t[1].clientY) / 2];
|
||||
@@ -293,7 +337,14 @@
|
||||
// Mouse: drag to move a zoomed page around; a click without dragging uses the tap zones.
|
||||
let drag = null, dragged = false;
|
||||
stage.addEventListener('pointerdown', e => {
|
||||
if (e.pointerType === 'mouse' && e.button === 0) drag = { x: e.clientX, y: e.clientY, l: stage.scrollLeft, t: stage.scrollTop, moved: false };
|
||||
if (e.pointerType === 'mouse' && e.button === 0) drag = {
|
||||
x: e.clientX,
|
||||
y: e.clientY,
|
||||
l: stage.scrollLeft,
|
||||
t: stage.scrollTop,
|
||||
panX: !lockFlowX && stage.scrollWidth > stage.clientWidth + 1,
|
||||
moved: false
|
||||
};
|
||||
});
|
||||
addEventListener('pointermove', e => {
|
||||
if (!drag) return;
|
||||
@@ -301,7 +352,7 @@
|
||||
if (!drag.moved && Math.abs(dx) + Math.abs(dy) < 6) return;
|
||||
drag.moved = true;
|
||||
stage.classList.add('drag');
|
||||
stage.scrollLeft = drag.l - dx;
|
||||
if (drag.panX) stage.scrollLeft = drag.l - dx;
|
||||
stage.scrollTop = drag.t - dy;
|
||||
});
|
||||
addEventListener('pointerup', () => {
|
||||
@@ -338,8 +389,8 @@
|
||||
if (k === 'ArrowLeft' || k === 'ArrowRight') { e.preventDefault(); (k === (rtl ? 'ArrowLeft' : 'ArrowRight') ? next : prev)(); }
|
||||
else if ((k === ' ' || k === 'PageDown') && mode !== 'flow') { e.preventDefault(); (e.shiftKey ? prev : next)(); }
|
||||
else if (k === 'PageUp' && mode !== 'flow') { e.preventDefault(); prev(); }
|
||||
else if (k === '+' || k === '=') setZoom(zoom() * 1.15);
|
||||
else if (k === '-') setZoom(zoom() / 1.15);
|
||||
else if (k === '+' || k === '=') stepZoom(1);
|
||||
else if (k === '-') stepZoom(-1);
|
||||
else if (k === '0') setZoom(1);
|
||||
else if (k === 'm' || k === 'Escape') toggleBars();
|
||||
else if (k === 'f') toggleFullscreen();
|
||||
@@ -347,6 +398,7 @@
|
||||
|
||||
let scrollRaf = 0;
|
||||
stage.addEventListener('scroll', () => {
|
||||
if (lockFlowX && stage.scrollLeft !== 0) stage.scrollLeft = 0;
|
||||
if (mode !== 'flow' || scrollRaf) return;
|
||||
scrollRaf = requestAnimationFrame(() => {
|
||||
scrollRaf = 0;
|
||||
@@ -485,6 +537,7 @@
|
||||
if (b && stage.scrollTop + stage.clientHeight >= b.y + b.h - 4) { done = true; cur = list.length - 1; }
|
||||
} else if (atEnd || (mode === 'page' ? cur === list.length - 1 : spreadIdx === spreads.length - 1)) done = true;
|
||||
updateUI();
|
||||
if (readingClock) readingClock.viewChanged();
|
||||
clearTimeout(saveTimer);
|
||||
saveTimer = setTimeout(saveProgress, 700);
|
||||
}
|
||||
@@ -519,7 +572,9 @@
|
||||
ver.onchange = () => { saveProgress(true); location.href = readUrl(D.manga.id, D.chapter.id, ver.value); };
|
||||
for (const b of modeBtns) b.onclick = () => { if (b.dataset.mode !== mode) setMode(b.dataset.mode, true); };
|
||||
$('#shift').onclick = toggleShift;
|
||||
$('#zoomout').onclick = () => stepZoom(-1);
|
||||
$('#zoom').onclick = () => setZoom(1);
|
||||
$('#zoomin').onclick = () => stepZoom(1);
|
||||
slider.dir = rtl ? 'rtl' : 'ltr';
|
||||
slider.addEventListener('input', () => goTo(+slider.value - 1));
|
||||
function toggleFullscreen() {
|
||||
@@ -533,6 +588,19 @@
|
||||
for (const bar of [$('#top'), $('#bot')]) bar.addEventListener('click', e => { if (e.target.closest('button')) stage.focus({ preventScroll: true }); });
|
||||
|
||||
setMode(mode, false);
|
||||
if (S.readingClock) readingClock = createReadingClock({
|
||||
button: $('#readclock'),
|
||||
panel: $('#clockpanel'),
|
||||
identity: { m: D.manga.id, c: D.chapter.id, v: D.version },
|
||||
initial: D.reading,
|
||||
eligibleCount: list.length,
|
||||
eligiblePages: list.map(page => page.i),
|
||||
surface: stage,
|
||||
view: readingView,
|
||||
onComplete: result => toast(result.completedPasses > 1
|
||||
? 'Reread time is now counted · 60% of pages viewed'
|
||||
: 'Reading time is now counted · 60% of pages viewed', 3500),
|
||||
});
|
||||
stage.focus({ preventScroll: true });
|
||||
setBars(true);
|
||||
})();
|
||||
@@ -0,0 +1,32 @@
|
||||
'use strict';
|
||||
// Shared pure policy. No speed limit; credit only loaded pages actually observed,
|
||||
// not pages skipped between two scroll positions. Coordinates never leave reader.
|
||||
(function (root) {
|
||||
const IDLE_MS = 30000, EXPOSURE_MS = 100;
|
||||
function exposureTracker() {
|
||||
const seen = new Set(), exposures = new Map();
|
||||
function update(pages, now, active) {
|
||||
const visible = new Set(active ? pages.filter(page => page.loaded).map(page => page.index) : []);
|
||||
for (const [page, entry] of exposures) {
|
||||
if (entry.visible) entry.ms += Math.max(0, now - entry.at);
|
||||
entry.at = now; entry.visible = visible.has(page);
|
||||
if (entry.ms >= EXPOSURE_MS) { seen.add(page); exposures.delete(page); }
|
||||
}
|
||||
for (const page of visible) if (!seen.has(page) && !exposures.has(page)) exposures.set(page, { ms: 0, at: now, visible: true });
|
||||
return seen;
|
||||
}
|
||||
return { seen, update };
|
||||
}
|
||||
function stateFor({ visible, focused, lastInputAt, now, owned, view }) {
|
||||
if (!visible) return 'hidden';
|
||||
if (!focused) return 'unfocused';
|
||||
if (now - lastInputAt >= IDLE_MS) return 'idle';
|
||||
if (!owned) return 'other-tab';
|
||||
if (view.atEnd) return 'finished';
|
||||
if (view.pages.some(page => page.loaded)) return 'reading';
|
||||
return view.pages.some(page => !page.failed) || !view.pages.length ? 'loading' : 'page-error';
|
||||
}
|
||||
const policy = { IDLE_MS, EXPOSURE_MS, exposureTracker, stateFor };
|
||||
if (typeof module !== 'undefined' && module.exports) module.exports = policy;
|
||||
else root.ReadingClockPolicy = policy;
|
||||
})(typeof window === 'undefined' ? globalThis : window);
|
||||
@@ -0,0 +1,177 @@
|
||||
'use strict';
|
||||
/* Opt-in, personal reading estimates. The server counts visits with 60% coverage. */
|
||||
(function () {
|
||||
const { IDLE_MS, exposureTracker, stateFor } = window.ReadingClockPolicy;
|
||||
const INPUTS = ['mouse', 'touch', 'keyboard'], PULSE_MS = 10000;
|
||||
const labels = {
|
||||
reading: ['⏱', 'Reading'], loading: ['⌛', 'Loading page'], idle: ['⏸', 'Idle'],
|
||||
hidden: ['⏸', 'Tab hidden'], unfocused: ['⏸', 'Window inactive'],
|
||||
'other-tab': ['⏸', 'Another reader is active'], 'page-error': ['⏸', 'Page unavailable'],
|
||||
finished: ['⏸', 'Chapter finished'],
|
||||
};
|
||||
const blank = () => ({ activeMs: 0, loadingMs: 0, inputMs: { mouse: 0, touch: 0, keyboard: 0 } });
|
||||
const fmt = ms => {
|
||||
const seconds = Math.max(0, Math.floor(ms / 1000)), hours = Math.floor(seconds / 3600);
|
||||
return (hours ? hours + ':' : '') + String(Math.floor(seconds / 60) % 60).padStart(2, '0') + ':' + String(seconds % 60).padStart(2, '0');
|
||||
};
|
||||
window.createReadingClock = function (options) {
|
||||
const button = options.button, panel = options.panel;
|
||||
const session = (crypto.randomUUID ? crypto.randomUUID() : Math.random().toString(36).slice(2) + Date.now().toString(36)).replace(/[^a-zA-Z0-9_-]/g, '');
|
||||
const started = performance.now(), eligible = new Set(options.eligiblePages);
|
||||
const required = Math.ceil(eligible.size * 0.6);
|
||||
let visit = blank(), acknowledged = blank(), tracker = exposureTracker(), acknowledgedSeen = [];
|
||||
let reading = options.initial || {}, state = 'idle', lastSample = started, lastInputAt = started, lastType = 'mouse';
|
||||
let focused = document.hasFocus(), needsInput = false, owned = true, claimAt = 0, channel;
|
||||
let sequence = 0, lastSent = started, sending = false, urgent = false, releasePending = false;
|
||||
let retryTimer = 0, saveError = false, ended = false, destroyed = false, notified = false;
|
||||
const listeners = [];
|
||||
function listen(target, name, fn, opts) {
|
||||
target.addEventListener(name, fn, opts); listeners.push(() => target.removeEventListener(name, fn, opts));
|
||||
}
|
||||
function accrue(now) {
|
||||
const delta = Math.max(0, Math.min(now, lastInputAt + IDLE_MS) - lastSample);
|
||||
if (state === 'reading') { visit.activeMs += delta; visit.inputMs[lastType] += delta; }
|
||||
else if (state === 'loading') visit.loadingMs += delta;
|
||||
lastSample = now;
|
||||
}
|
||||
function covered() { return [...tracker.seen].filter(page => eligible.has(page)); }
|
||||
function sample() {
|
||||
if (destroyed || ended) return;
|
||||
const now = performance.now(); accrue(now);
|
||||
const view = options.view();
|
||||
const previous = state;
|
||||
state = stateFor({ visible: !document.hidden, focused, now, lastInputAt: needsInput ? -Infinity : lastInputAt, owned, view });
|
||||
tracker.update(view.pages, now, state === 'reading');
|
||||
if (previous !== state && ['idle', 'finished'].includes(state) && ['reading', 'loading'].includes(previous)) requestSave(true);
|
||||
if (now - lastSent >= PULSE_MS && owned && ['reading', 'loading'].includes(state)) requestSave();
|
||||
render();
|
||||
}
|
||||
function snapshot(final = false, release = false) {
|
||||
lastSent = performance.now();
|
||||
return { ...options.identity, session, seq: ++sequence, elapsedMs: Math.round(lastSent - started),
|
||||
activeMs: Math.round(visit.activeMs), loadingMs: Math.round(visit.loadingMs),
|
||||
inputMs: Object.fromEntries(INPUTS.map(key => [key, Math.round(visit.inputMs[key])])),
|
||||
seen: covered(), final, release: final || release };
|
||||
}
|
||||
function requestSave(release = false) {
|
||||
urgent = true; releasePending ||= release;
|
||||
pump();
|
||||
}
|
||||
async function pump() {
|
||||
if (sending || destroyed || ended || !urgent) return;
|
||||
sending = true; urgent = false;
|
||||
const body = snapshot(false, releasePending); releasePending = false;
|
||||
clearTimeout(retryTimer);
|
||||
try {
|
||||
const response = await fetch('api/reading/visit', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || response.statusText);
|
||||
saveError = false;
|
||||
if (data.accepted) {
|
||||
acknowledged = body; acknowledgedSeen = body.seen; reading = data.reading || reading;
|
||||
if (reading.currentPass?.complete && !notified) {
|
||||
notified = true;
|
||||
options.onComplete?.({ activeMs: visit.activeMs, totalActiveMs: reading.totalActiveMs, completedPasses: reading.completedPasses });
|
||||
}
|
||||
} else {
|
||||
accrue(performance.now()); owned = false; state = 'other-tab';
|
||||
visit = { ...acknowledged, inputMs: { ...acknowledged.inputMs } };
|
||||
tracker = exposureTracker(); acknowledgedSeen.forEach(page => tracker.seen.add(page));
|
||||
retryTimer = setTimeout(() => { if (!needsInput && !ended) { owned = true; requestSave(); sample(); } }, Math.max(500, data.retryAfterMs || 2500));
|
||||
}
|
||||
} catch {
|
||||
// Network delays aren't fake reading. Retain cumulative time and retry.
|
||||
saveError = true;
|
||||
retryTimer = setTimeout(() => requestSave(), 2500);
|
||||
} finally {
|
||||
sending = false; sample();
|
||||
// New milliseconds accrued during a response must NOT trigger another
|
||||
// request. Only an explicitly requested save can run the pump again.
|
||||
if (urgent) pump();
|
||||
}
|
||||
}
|
||||
function flush() {
|
||||
if (destroyed || ended) return;
|
||||
sample(); ended = true;
|
||||
const body = JSON.stringify(snapshot(true, true));
|
||||
// Send even with a pulse in flight. Cumulative sequence handling makes
|
||||
// either arrival order safe, including the final beacon arriving first.
|
||||
let queued = false;
|
||||
try { queued = navigator.sendBeacon?.('api/reading/visit', new Blob([body], { type: 'application/json' })); } catch {}
|
||||
if (!queued) fetch('api/reading/visit', { method: 'POST', keepalive: true, headers: { 'Content-Type': 'application/json' }, body }).catch(() => {});
|
||||
}
|
||||
function input(type) {
|
||||
if (ended || destroyed || document.hidden) return;
|
||||
const now = performance.now(); accrue(now);
|
||||
lastType = type; lastInputAt = now; needsInput = false; focused = document.hasFocus();
|
||||
if (!owned) { owned = true; requestSave(); }
|
||||
const at = Date.now();
|
||||
if (at - claimAt >= 1000) { claimAt = at; channel?.postMessage({ session, at }); }
|
||||
sample();
|
||||
}
|
||||
function render() {
|
||||
const [icon, label] = labels[state] || labels.idle, coverage = covered().length;
|
||||
const qualified = coverage >= required && visit.activeMs > 0;
|
||||
const total = (+reading.totalActiveMs || 0) + (qualified ? Math.max(0, visit.activeMs - (reading.currentPass?.complete ? acknowledged.activeMs : 0)) : 0);
|
||||
button.querySelector('.clock-icon').textContent = icon;
|
||||
button.querySelector('.clock-time').textContent = fmt(visit.activeMs);
|
||||
button.querySelector('.clock-label').textContent = ` · ${label}${saveError ? ' · Save pending' : ''}`;
|
||||
button.classList.toggle('clock-running', state === 'reading');
|
||||
button.classList.remove('clock-warning');
|
||||
button.title = `Active Reading Clock · ${label} · ${coverage}/${eligible.size} pages`;
|
||||
if (!panel.hidden) {
|
||||
panel.textContent = '';
|
||||
panel.append(h('div', { class: 'clock-head' }, h('b', {}, 'Active Reading Clock'), h('span', { class: 'clock-state' }, `${icon} ${label}`)),
|
||||
h('dl', { class: 'clock-details' },
|
||||
h('dt', {}, 'This visit'), h('dd', {}, fmt(visit.activeMs)),
|
||||
h('dt', {}, 'Pages viewed'), h('dd', {}, `${coverage} / ${eligible.size}`),
|
||||
h('dt', {}, 'Counted as a read'), h('dd', {}, qualified ? 'Yes · time continues counting' : `Not yet · ${required} pages needed (60%)`),
|
||||
h('dt', {}, 'Saved chapter total'), h('dd', {}, fmt(total)),
|
||||
h('dt', {}, 'Loading wait'), h('dd', {}, fmt(visit.loadingMs))),
|
||||
h('p', { class: 'muted small' }, saveError ? 'Save pending. Your time is retained in this tab and will be retried.' :
|
||||
'No scrolling-speed limit. Loaded pages count after 100 ms of visible exposure. At 60% coverage, all active time from this visit counts and is saved when you leave. Short picture lookups stay out of your totals. Pauses in the background and after 30 seconds without reader activity.'));
|
||||
}
|
||||
}
|
||||
button.hidden = false;
|
||||
listen(button, 'click', e => { e.stopPropagation(); panel.hidden = !panel.hidden; render(); });
|
||||
listen(panel, 'click', e => e.stopPropagation());
|
||||
listen(document, 'click', () => { panel.hidden = true; });
|
||||
try {
|
||||
channel = new BroadcastChannel('mangacircus-reading-clock');
|
||||
channel.onmessage = event => {
|
||||
const msg = event.data || {};
|
||||
if (msg.session === session || !msg.at || msg.at < claimAt || (msg.at === claimAt && msg.session < session)) return;
|
||||
accrue(performance.now()); owned = false; state = 'other-tab'; requestSave(true); render();
|
||||
};
|
||||
} catch { channel = null; }
|
||||
let pointer = null;
|
||||
listen(document, 'pointerdown', e => { if (e.isTrusted) input(e.pointerType === 'touch' ? 'touch' : 'mouse'); }, { passive: true });
|
||||
listen(document, 'pointermove', e => {
|
||||
if (!e.isTrusted) return;
|
||||
const now = performance.now();
|
||||
if (!pointer || (now - pointer.at >= 150 && Math.hypot(e.clientX - pointer.x, e.clientY - pointer.y) >= 3)) {
|
||||
pointer = { x: e.clientX, y: e.clientY, at: now }; input(e.pointerType === 'touch' ? 'touch' : 'mouse');
|
||||
}
|
||||
}, { passive: true });
|
||||
listen(document, 'wheel', e => { if (e.isTrusted) input('mouse'); }, { passive: true });
|
||||
listen(document, 'keydown', e => {
|
||||
if (e.isTrusted && ['ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', ' ', 'PageDown', 'PageUp', 'Home', 'End', '+', '=', '-', '0', 'm', 'Escape', 'f'].includes(e.key)) input('keyboard');
|
||||
});
|
||||
// Momentum scrolling remains activity after the finger lifts.
|
||||
if (options.surface) listen(options.surface, 'scroll', e => { if (e.isTrusted && !needsInput) input(lastType); }, { passive: true });
|
||||
listen(window, 'blur', () => { accrue(performance.now()); focused = false; needsInput = true; sample(); requestSave(true); });
|
||||
listen(window, 'focus', () => { focused = true; sample(); });
|
||||
listen(document, 'visibilitychange', () => {
|
||||
if (document.hidden) { accrue(performance.now()); needsInput = true; sample(); requestSave(true); } else sample();
|
||||
});
|
||||
listen(window, 'pagehide', flush);
|
||||
listen(document, 'freeze', () => { accrue(performance.now()); needsInput = true; sample(); requestSave(true); });
|
||||
listen(window, 'pageshow', e => { if (e.persisted) { ended = false; lastSample = performance.now(); input(lastType); requestSave(); } });
|
||||
const timer = setInterval(sample, 50);
|
||||
sample(); requestSave();
|
||||
return { viewChanged: sample, flush, destroy() {
|
||||
flush(); destroyed = true; clearInterval(timer); clearTimeout(retryTimer);
|
||||
listeners.forEach(remove => remove()); channel?.close();
|
||||
} };
|
||||
};
|
||||
})();
|
||||
@@ -0,0 +1,60 @@
|
||||
'use strict';
|
||||
|
||||
// Two navigation levels; scraper tabs stay within their section, not another menu.
|
||||
const SETTINGS_SECTIONS = [
|
||||
{ id: 'user/reading', group: 'User', title: 'Reading', hint: 'Reader modes, page spacing and zoom.' },
|
||||
{ id: 'user/reading-time', group: 'User', title: 'Reading time & privacy', hint: 'Your clock, Reading Story and personal data.' },
|
||||
{ id: 'user/notifications', group: 'User', title: 'Notifications', hint: 'Chapter alerts and your optional webhook.' },
|
||||
{ id: 'user/account', group: 'User', title: 'Account', hint: 'Your account and password.' },
|
||||
{ id: 'admin/scraper', group: 'Admin', title: 'Scraper', hint: 'Live activity, downloads, update checks and sources.' },
|
||||
{ id: 'admin/library', group: 'Admin', title: 'Server & library', hint: 'Library location, scan status and maintenance.' },
|
||||
{ id: 'admin/users', group: 'Admin', title: 'Users & access', hint: 'Manage accounts, passwords and administrators.' },
|
||||
{ id: 'admin/collections', group: 'Admin', title: 'Collections & metadata', hint: 'Group manga and review their relationships.', href: 'collections.html' },
|
||||
{ id: 'admin/health', group: 'Admin', title: 'Service health', hint: 'Service status, incidents and recovery.', href: 'health.html' },
|
||||
];
|
||||
const SCRAPER_TABS = ['activity', 'queue', 'updates', 'sources'];
|
||||
|
||||
function settingsRoute(hash, admin) {
|
||||
const parts = String(hash || '').replace(/^#/, '').split('/');
|
||||
const candidate = parts.slice(0, 2).join('/'), tab = parts[2];
|
||||
const section = SETTINGS_SECTIONS.find(item => item.id === candidate && (admin || item.group === 'User'));
|
||||
return { id: section?.id || 'user/reading', tab: SCRAPER_TABS.includes(tab) ? tab : tab === 'hosters' ? 'sources' : 'activity' };
|
||||
}
|
||||
|
||||
function settingsShell(me, active = 'user/reading') {
|
||||
const main = $('main'); main.textContent = '';
|
||||
const nav = h('nav', { class: 'settings-menu', 'aria-label': 'Settings sections' });
|
||||
const links = new Map();
|
||||
for (const group of me.admin ? ['User', 'Admin'] : ['User']) {
|
||||
nav.append(h('div', { class: 'settings-menu-group' }, h('h2', {}, group),
|
||||
SETTINGS_SECTIONS.filter(section => section.group === group).map(section => {
|
||||
const link = h('a', { href: section.href || 'settings.html#' + section.id }, section.title);
|
||||
links.set(section.id, link);
|
||||
return link;
|
||||
})));
|
||||
}
|
||||
const crumb = h('p', { class: 'settings-breadcrumb muted small' });
|
||||
const title = h('h1', { tabindex: '-1' });
|
||||
const titleRow = h('div', { class: 'settings-title-row' }, title);
|
||||
const hint = h('p', { class: 'muted settings-hint' });
|
||||
const content = h('div', { class: 'settings-content' });
|
||||
main.append(h('div', { class: 'settings-layout' }, nav,
|
||||
h('div', { class: 'settings-workspace' }, crumb, titleRow, hint, content)));
|
||||
function select(id, focus = false) {
|
||||
const section = SETTINGS_SECTIONS.find(item => item.id === id);
|
||||
if (!section || (section.group === 'Admin' && !me.admin)) return;
|
||||
for (const [key, link] of links) {
|
||||
if (key === id) link.setAttribute('aria-current', 'page');
|
||||
else link.removeAttribute('aria-current');
|
||||
}
|
||||
crumb.textContent = 'Settings / ' + section.group;
|
||||
title.textContent = section.title;
|
||||
hint.textContent = section.hint;
|
||||
document.title = section.title + ' · Settings · MangaCircus';
|
||||
if (focus) title.focus({ preventScroll: true });
|
||||
}
|
||||
select(active);
|
||||
return { content, titleRow, select };
|
||||
}
|
||||
|
||||
if (typeof module !== 'undefined') module.exports = { SETTINGS_SECTIONS, SCRAPER_TABS, settingsRoute };
|
||||
@@ -9,6 +9,7 @@
|
||||
<link rel="manifest" href="manifest.webmanifest">
|
||||
<meta name="theme-color" content="#121214">
|
||||
<script src="common.js" defer></script>
|
||||
<script src="settings-nav.js" defer></script>
|
||||
<script src="settings.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
+360
-13
@@ -1,13 +1,19 @@
|
||||
'use strict';
|
||||
/* Settings: reading preferences, notifications, password and (for admins) library + users. */
|
||||
/* Settings workspace: User/Admin sections, with local scraper tabs. */
|
||||
(async function () {
|
||||
let me;
|
||||
try { me = await api('api/me'); } catch (e) { return showError(e); }
|
||||
topNav(me);
|
||||
const S = me.settings, main = $('main');
|
||||
main.textContent = '';
|
||||
const S = me.settings, shell = settingsShell(me);
|
||||
const sections = new Map(), controllers = new Map();
|
||||
const section = id => {
|
||||
if (!sections.has(id)) sections.set(id, h('div', { 'data-settings-section': id }));
|
||||
return sections.get(id);
|
||||
};
|
||||
let route = settingsRoute(location.hash, me.admin);
|
||||
|
||||
const saved = h('span', { class: 'saved', hidden: true }, 'Saved');
|
||||
shell.titleRow.append(saved);
|
||||
let savedTimer = 0;
|
||||
async function save(patch) {
|
||||
Object.assign(S, (await api('api/settings', patch)).settings);
|
||||
@@ -28,22 +34,59 @@
|
||||
const LABEL = { flow: 'Flow', page: 'Page', book: 'Book', strip: 'Webtoon flow' };
|
||||
const zooms = () => Object.keys(LABEL).map(k => `${LABEL[k]} ${Math.round((S.zoom[k] || 1) * 100)}%`).join(' · ');
|
||||
const zoomText = h('span', { class: 'muted' }, zooms());
|
||||
const readingPassword = h('input', { type: 'password', autocomplete: 'current-password', placeholder: 'Current password' });
|
||||
const readingDataMessage = h('span', { class: 'small' });
|
||||
const readingData = h('details', { class: 'reading-data' },
|
||||
h('summary', {}, 'Manage my reading-time data'),
|
||||
h('p', { class: 'muted small' }, 'Reading Story is private to your account. Turning the clock off stops collection but keeps existing history.'),
|
||||
h('div', { class: 'row' },
|
||||
h('button', { class: 'btn small', onclick: async () => {
|
||||
try {
|
||||
const data = await api('api/reading/export');
|
||||
const blob = new Blob([JSON.stringify(data, null, 2) + '\n'], { type: 'application/json' });
|
||||
const link = h('a', { href: URL.createObjectURL(blob), download: `mangacircus-reading-${me.name}.json` });
|
||||
document.body.append(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(link.href), 0);
|
||||
readingDataMessage.textContent = 'Export created'; readingDataMessage.className = 'saved';
|
||||
} catch (error) { readingDataMessage.textContent = error.message; readingDataMessage.className = 'small error'; }
|
||||
} }, 'Export my summary')),
|
||||
h('div', { class: 'reading-danger' },
|
||||
h('b', {}, 'Delete all reading-time history'),
|
||||
h('p', { class: 'muted small' }, 'Reading progress and accounts are not affected. This cannot be undone.'),
|
||||
h('div', { class: 'row' }, readingPassword,
|
||||
h('button', { class: 'btn small danger', onclick: async () => {
|
||||
if (!readingPassword.value || !confirm('Permanently delete all of your reading-time history?')) return;
|
||||
try {
|
||||
await api('api/reading/delete', { all: true, password: readingPassword.value });
|
||||
readingPassword.value = '';
|
||||
readingDataMessage.textContent = 'Reading-time history deleted'; readingDataMessage.className = 'saved';
|
||||
} catch (error) { readingDataMessage.textContent = error.message; readingDataMessage.className = 'small error'; }
|
||||
} }, 'Delete history'), readingDataMessage)));
|
||||
|
||||
main.append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Reading ', saved),
|
||||
section('user/reading').append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Reader preferences'),
|
||||
h('label', { class: 'field' }, h('span', {}, 'Open chapters in'), startMode,
|
||||
h('div', { class: 'muted small' }, `Last used: ${LABEL[S.lastMode]}. Only clicking a mode in the reader changes it.`)),
|
||||
check('adapt', 'Use the fitting mode when the content is not made for my choice',
|
||||
'Example: webtoons are one long strip, so they always open in Flow. This never changes the mode you picked.'),
|
||||
check('skipExtras', 'Skip extra pages and extra chapters', 'Pages or chapters marked as extra (omake, notes, specials) are left out while reading.'),
|
||||
check('skipCovers', 'Skip chapter cover pages'),
|
||||
check('flowPageGap', 'Show a gap between pages in Flow mode',
|
||||
'Turn this off for continuous webtoon artwork or text that crosses page edges.'),
|
||||
h('div', { class: 'field' }, h('span', {}, 'Page zoom'), zoomText, ' ',
|
||||
h('button', { class: 'btn small', onclick: async () => { await save({ zoom: {} }); zoomText.textContent = zooms(); } }, 'Reset to fit'),
|
||||
h('div', { class: 'muted small' }, '100% = the page fits the screen. Change it in the reader with Ctrl + mouse wheel, a pinch, or the + / - keys.'))));
|
||||
|
||||
section('user/reading-time').append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Clock & Reading Story'),
|
||||
check('readingClock', 'Track and show the Active Reading Clock',
|
||||
'Counts a visit after you view 60% of its pages, with no scrolling-speed limit. Pauses in the background or after 30 seconds without reader activity. Disabled by default.'),
|
||||
check('readingStory', 'Show Your Reading Story',
|
||||
'Shows your private chapter times and personal summaries on manga and library pages. Hiding it does not delete existing history.'),
|
||||
readingData));
|
||||
|
||||
const hook = h('input', { type: 'url', placeholder: 'https://ntfy.sh/my-manga-topic', value: S.webhook || '', style: 'width:100%' });
|
||||
const hookMsg = h('div', { class: 'small' });
|
||||
main.append(h('section', { class: 'card' },
|
||||
section('user/notifications').append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Notifications'),
|
||||
h('p', { class: 'muted' }, 'Press "Notify me" on a manga page. New chapters then show up under the bell at the top.'),
|
||||
h('label', { class: 'field' }, h('span', {}, 'Also send them to a webhook (optional)'), hook,
|
||||
@@ -55,18 +98,290 @@
|
||||
const oldPw = h('input', { type: 'password', autocomplete: 'current-password', placeholder: 'Current password' });
|
||||
const newPw = h('input', { type: 'password', autocomplete: 'new-password', placeholder: 'New password' });
|
||||
const pwMsg = h('span', { class: 'small' });
|
||||
main.append(h('section', { class: 'card' },
|
||||
section('user/account').append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Account · ' + me.name),
|
||||
h('div', { class: 'row' }, oldPw, newPw, h('button', { class: 'btn', onclick: async () => {
|
||||
try { await api('api/password', { old: oldPw.value, new: newPw.value }); pwMsg.textContent = 'Password changed'; pwMsg.className = 'saved'; oldPw.value = newPw.value = ''; }
|
||||
catch (e) { pwMsg.textContent = e.message; pwMsg.className = 'small error'; }
|
||||
} }, 'Change password'), pwMsg)));
|
||||
|
||||
if (me.admin) adminCard();
|
||||
function scraperCard() {
|
||||
const card = h('section', { class: 'card scraper-card' });
|
||||
section('admin/scraper').append(card);
|
||||
let timer = 0, active = false, lastState = null, dirty = false, sourceFilter = '', onlyEnabled = false;
|
||||
const disclosureOpen = { add: false, completed: false, history: false };
|
||||
const commandError = h('p', { class: 'error small', role: 'alert', hidden: true });
|
||||
card.addEventListener('input', () => { dirty = true; });
|
||||
// A poll can resolve between the summary click and the browser's deferred
|
||||
// details toggle event. Record the intended state synchronously, rather
|
||||
// than sampling the old DOM immediately before replacing it.
|
||||
card.addEventListener('click', event => {
|
||||
const summary = event.target.closest('summary');
|
||||
const details = summary?.parentElement;
|
||||
const key = details?.dataset.scraperDisclosure;
|
||||
if (key && Object.hasOwn(disclosureOpen, key)) disclosureOpen[key] = !details.open;
|
||||
});
|
||||
card.addEventListener('toggle', event => {
|
||||
const details = event.target;
|
||||
const key = details.dataset?.scraperDisclosure;
|
||||
if (details.isConnected && key && Object.hasOwn(disclosureOpen, key)) disclosureOpen[key] = details.open;
|
||||
}, true);
|
||||
|
||||
async function adminCard() {
|
||||
const command = async payload => {
|
||||
try {
|
||||
await api('api/admin/scraper', payload);
|
||||
dirty = false; commandError.hidden = true;
|
||||
await load(true);
|
||||
} catch (e) { commandError.textContent = e.message; commandError.hidden = false; }
|
||||
};
|
||||
|
||||
const statusText = entry => {
|
||||
const job = entry.job;
|
||||
const bits = [entry.current.title];
|
||||
if (job?.current?.book?.title) bits.push(job.current.book.title);
|
||||
if (job?.current?.number) bits.push(job.current.label || 'chapter ' + job.current.number);
|
||||
if (job?.current?.expectedPages) bits.push(`page ${job.current.downloadedPages || 0}/${job.current.expectedPages}`);
|
||||
return bits.join(' · ');
|
||||
};
|
||||
|
||||
const draw = state => {
|
||||
lastState = state;
|
||||
card.textContent = '';
|
||||
const tabs = h('nav', { class: 'settings-tabs', 'aria-label': 'Scraper sections' },
|
||||
[['activity', 'Activity'], ['queue', 'Queue'], ['updates', 'Updates'], ['sources', 'Sources']].map(([id, label]) =>
|
||||
h('a', { class: 'btn', href: '#admin/scraper/' + id, 'aria-current': route.tab === id ? 'page' : null }, label)));
|
||||
card.append(tabs, commandError);
|
||||
if (!state.available) {
|
||||
card.append(h('h2', {}, 'Scraper'),
|
||||
h('p', { class: 'error' }, state.error || 'Scraper service unavailable'),
|
||||
h('p', { class: 'muted small' }, 'Check the managed services and their logs. Do not start a second scraper against the same library.'),
|
||||
h('a', { class: 'btn small', href: 'health.html' }, 'Service health & incidents'));
|
||||
return;
|
||||
}
|
||||
const activeEntries = state.active?.length ? state.active : state.current ? [{ source: state.current.source, current: state.current, job: state.job }] : [];
|
||||
const url = h('input', { type: 'url', placeholder: 'WeebCentral or MangaDex title URL', class: 'grow' });
|
||||
const title = h('input', { placeholder: 'Title override (optional)' });
|
||||
const scope = h('select', {},
|
||||
h('option', { value: 'missing' }, 'All missing chapters'),
|
||||
h('option', { value: 'latest' }, 'Newest chapter only'),
|
||||
h('option', { value: 'last' }, 'Last N chapters'),
|
||||
h('option', { value: 'first' }, 'First N chapters'),
|
||||
h('option', { value: 'chapter' }, 'One chapter'),
|
||||
h('option', { value: 'after' }, 'After chapter'),
|
||||
h('option', { value: 'volume' }, 'One volume'));
|
||||
const scopeValue = h('input', { type: 'number', min: '0', step: '0.1', placeholder: 'N / chapter / volume', disabled: true });
|
||||
scope.onchange = () => { scopeValue.disabled = ['missing', 'latest'].includes(scope.value); scopeValue.placeholder = ['last', 'first'].includes(scope.value) ? 'Count' : scope.value === 'volume' ? 'Volume' : 'Chapter'; };
|
||||
const addSource = h('input', { type: 'checkbox' });
|
||||
const libraryTarget = h('select', {}, h('option', { value: '' }, 'Auto-detect library manga'),
|
||||
...(state.library || []).map(manga => h('option', { value: manga.id }, manga.title)));
|
||||
const addMessage = h('span', { class: 'small' });
|
||||
const rate = state.settings || {};
|
||||
const sourceRateEditors = (state.adapters || []).map(adapter => {
|
||||
const sourceRate = { ...rate, ...((rate.sourceRates || {})[adapter.id] || {}) };
|
||||
const min = h('input', { type: 'number', min: '.5', step: '.5', value: (sourceRate.minMs ?? 1000) / 1000, title: 'Minimum seconds between image and API requests' });
|
||||
const max = h('input', { type: 'number', min: '.5', step: '.5', value: (sourceRate.maxMs ?? 4000) / 1000, title: 'Maximum seconds between image and API requests' });
|
||||
const chapterMin = h('input', { type: 'number', min: '.5', step: '1', value: (sourceRate.chapterMinMs ?? 5000) / 1000, title: 'Minimum seconds between chapters' });
|
||||
const chapterMax = h('input', { type: 'number', min: '.5', step: '1', value: (sourceRate.chapterMaxMs ?? 15000) / 1000, title: 'Maximum seconds between chapters' });
|
||||
return h('div', { class: 'source-rate' },
|
||||
h('b', {}, adapter.label),
|
||||
h('div', { class: 'row rate-row' },
|
||||
h('label', {}, h('small', {}, 'Requests min (s)'), min), h('label', {}, h('small', {}, 'Requests max (s)'), max),
|
||||
h('label', {}, h('small', {}, 'Chapters min (s)'), chapterMin), h('label', {}, h('small', {}, 'Chapters max (s)'), chapterMax),
|
||||
h('button', { class: 'btn', onclick: () => command({ command: 'settings', settings: { sourceRates: { [adapter.id]: {
|
||||
minMs: +min.value * 1000, maxMs: +max.value * 1000,
|
||||
chapterMinMs: +chapterMin.value * 1000, chapterMaxMs: +chapterMax.value * 1000,
|
||||
} } } }) }, `Save ${adapter.label}`)));
|
||||
});
|
||||
const autoEnabled = h('input', { type: 'checkbox', checked: rate.autoUpdateEnabled !== false });
|
||||
const autoTime = h('input', { type: 'time', value: rate.autoUpdateTime || '03:00' });
|
||||
|
||||
const visible = items => items.filter(item => !sourceFilter || item.source === sourceFilter);
|
||||
const pending = visible(state.queue.filter(item => ['queued', 'retry_wait'].includes(item.status)));
|
||||
const attention = visible(state.queue.filter(item => ['ready', 'error', 'quarantined', 'unavailable', 'skipped'].includes(item.status)));
|
||||
const completed = visible(state.queue.filter(item => item.status === 'complete' && item.kind === 'download'));
|
||||
const when = value => value && !Number.isNaN(Date.parse(value)) ? new Date(value).toLocaleString() : 'Never';
|
||||
const chips = h('div', { class: 'row scraper-filters', role: 'group', 'aria-label': 'Filter by source' },
|
||||
[{ id: '', label: 'All sources' }, ...(state.adapters || []).map(adapter => ({ id: adapter.id, label: adapter.label }))].map(option =>
|
||||
h('button', { class: 'btn small', type: 'button', 'aria-pressed': sourceFilter === option.id ? 'true' : 'false',
|
||||
onclick: () => { sourceFilter = option.id; draw(lastState); } }, option.label)));
|
||||
card.append(chips);
|
||||
const move = async (item, delta) => {
|
||||
const position = pending.findIndex(entry => entry.id === item.id), other = pending[position + delta];
|
||||
if (!other) return;
|
||||
const ids = state.queue.map(entry => entry.id), a = ids.indexOf(item.id), b = ids.indexOf(other.id);
|
||||
[ids[a], ids[b]] = [ids[b], ids[a]];
|
||||
await command({ command: 'reorder', ids });
|
||||
};
|
||||
const jobRow = (item, movable = false) => {
|
||||
const running = activeEntries.some(entry => entry.current.id === item.id);
|
||||
const availability = item.availability;
|
||||
const outcome = item.status === 'complete'
|
||||
? `${item.publishedCount ?? 0} chapter${item.publishedCount === 1 ? '' : 's'} published · finished ${when(item.completed)}`
|
||||
: item.status === 'ready' ? 'Awaiting edition confirmation' : item.status.replaceAll('_', ' ');
|
||||
return h('div', { class: 'scraper-item' + (running ? ' active' : ''), 'data-source': item.source },
|
||||
h('div', { class: 'grow' }, h('b', {}, item.title),
|
||||
h('div', { class: 'muted small' }, `${item.source} · ${item.kind === 'discovery' ? 'Checking for updates' : item.scopeLabel || 'all missing chapters'} · ${outcome}`
|
||||
+ (item.existingPolicy === 'add-source' ? ' · alternate source' : '')),
|
||||
item.error ? h('div', { class: 'small error' }, item.error) : null,
|
||||
item.sourceTitle ? h('div', { class: 'small' }, `Source edition: ${item.sourceTitle}`) : null,
|
||||
availability ? h('div', { class: 'muted small' }, `${availability.selectedChapters} hosted selections${availability.specialReleases ? ` · ${availability.specialReleases} unnumbered releases` : ''}${availability.expectedLastChapter ? ` · metadata last chapter: ${availability.expectedLastChapter}` : ''} · ${availability.coverage === 'partial' ? 'Partial source coverage' : 'Whole-series completeness not verified'}`) : null),
|
||||
movable ? h('button', { class: 'btn small', disabled: running || pending[0]?.id === item.id, title: 'Move up', onclick: () => move(item, -1) }, '↑') : null,
|
||||
movable ? h('button', { class: 'btn small', disabled: running || pending.at(-1)?.id === item.id, title: 'Move down', onclick: () => move(item, 1) }, '↓') : null,
|
||||
item.status === 'error' ? h('button', { class: 'btn small primary', onclick: () => command({ command: 'control', action: 'retry', id: item.id }) }, 'Retry') : null,
|
||||
['unavailable', 'quarantined', 'complete', 'ready'].includes(item.status) ? h('button', { class: 'btn small', onclick: () => command({ command: 'control', action: 'inspect', id: item.id }) }, 'Inspect again') : null,
|
||||
item.status === 'ready' ? h('button', { class: 'btn small primary', onclick: () => command({ command: 'control', action: 'confirm', id: item.id }) }, 'Confirm download') : null,
|
||||
h('button', { class: 'btn small', disabled: running, onclick: async () => {
|
||||
if (confirm(`Remove ${item.title} from the queue? Its library chapters will stay.`)) await command({ command: 'remove', id: item.id });
|
||||
} }, 'Remove'));
|
||||
};
|
||||
const activeDownloads = h('div', { class: 'scraper-queue' });
|
||||
const activeVisible = activeEntries.filter(entry => !sourceFilter || entry.source === sourceFilter || entry.current?.source === sourceFilter);
|
||||
if (!activeVisible.length) activeDownloads.append(h('p', { class: 'muted small' }, state.paused ? 'Queue paused' : 'No active download on this source'));
|
||||
activeVisible.forEach(entry => {
|
||||
const job = entry.job, percent = job?.percent || 0;
|
||||
const problem = job?.lastError || job?.lastProblem;
|
||||
activeDownloads.append(h('div', { class: 'scraper-item active' }, h('div', { class: 'grow' },
|
||||
h('b', {}, entry.source || entry.current.source),
|
||||
h('div', {}, statusText(entry)),
|
||||
job?.total ? h('div', { class: 'scraper-progress', title: `${job.completed}/${job.total}` }, h('i', { style: `width:${percent}%` })) : null,
|
||||
job?.total ? h('div', { class: 'muted small' }, `${job.completed}/${job.total} chapters · ${percent}%`) : null,
|
||||
problem ? h('div', { class: problem.level === 'warning' ? 'small' : 'small error' }, problem.message || String(problem)) : null)));
|
||||
});
|
||||
const activityPanel = h('div', { 'data-scraper-tab': 'activity', hidden: route.tab !== 'activity' },
|
||||
h('h2', {}, 'Activity ', h('span', { class: 'tag' }, state.status)),
|
||||
h('div', { class: 'scraper-overview' },
|
||||
h('span', {}, `${activeVisible.length} active`), h('span', {}, `${pending.length} pending`),
|
||||
h('span', {}, `${attention.length} need attention`)),
|
||||
h('div', { class: 'field' }, h('span', {}, 'Downloading now'), activeDownloads,
|
||||
h('div', { class: 'row', style: 'margin-top:8px' }, state.paused
|
||||
? h('button', { class: 'btn primary', onclick: () => command({ command: 'control', action: 'resume' }) }, 'Resume queue')
|
||||
: h('button', { class: 'btn', onclick: () => command({ command: 'control', action: 'pause' }) }, 'Pause queue'))),
|
||||
h('div', { class: 'field' }, h('span', {}, `Needs attention · ${attention.length}`),
|
||||
attention.length ? h('div', { class: 'scraper-queue' }, attention.map(item => jobRow(item)))
|
||||
: h('p', { class: 'muted small' }, 'Nothing needs action on this source.'),
|
||||
h('a', { class: 'btn small', href: 'health.html' }, 'Service health & incidents')),
|
||||
h('div', { class: 'field' }, h('span', {}, 'Up next'), pending.filter(item => item.status !== 'ready').length
|
||||
? h('div', { class: 'scraper-queue' }, pending.filter(item => item.status !== 'ready').slice(0, 5).map(item => jobRow(item)))
|
||||
: h('p', { class: 'muted small' }, 'No downloads waiting.')));
|
||||
const addPanel = h('details', { class: 'scraper-add', 'data-scraper-disclosure': 'add', open: disclosureOpen.add }, h('summary', {}, 'Add manga to queue'),
|
||||
h('div', { class: 'field' },
|
||||
h('div', { class: 'row' }, url, title),
|
||||
h('div', { class: 'row', style: 'margin-top:8px' }, scope, scopeValue, libraryTarget,
|
||||
h('label', { class: 'row small' }, addSource, 'Add as another source version'),
|
||||
h('button', { class: 'btn primary', onclick: async () => {
|
||||
try {
|
||||
const selectedScope = { mode: scope.value };
|
||||
if (['last', 'first'].includes(scope.value)) selectedScope.count = +scopeValue.value;
|
||||
if (['chapter', 'after'].includes(scope.value)) selectedScope.chapter = scopeValue.value;
|
||||
if (scope.value === 'volume') selectedScope.volume = scopeValue.value;
|
||||
await command({
|
||||
command: 'enqueue', url: url.value, title: title.value, scope: selectedScope,
|
||||
existingPolicy: addSource.checked ? 'add-source' : 'skip-any',
|
||||
libraryMangaId: libraryTarget.value || undefined,
|
||||
inspect: true,
|
||||
});
|
||||
}
|
||||
catch (e) { addMessage.textContent = e.message; addMessage.className = 'small error'; }
|
||||
} }, 'Inspect & queue'), addMessage),
|
||||
h('div', { class: 'muted small' }, 'Inspection uses the normal source lane and request delays. Confirm the source edition before images are downloaded.'),
|
||||
h('div', { class: 'muted small' }, 'Existing chapters are never overwritten. Enable alternate versions only when you want the same chapter from this additional host.')));
|
||||
const queuePanel = h('div', { 'data-scraper-tab': 'queue', hidden: route.tab !== 'queue' },
|
||||
h('h2', {}, 'Queue'), addPanel,
|
||||
h('div', { class: 'field' }, h('span', {}, `Pending · ${pending.length}`),
|
||||
pending.length ? h('div', { class: 'scraper-queue' }, pending.map(item => jobRow(item, true)))
|
||||
: h('p', { class: 'muted small' }, 'No pending downloads on this source.')),
|
||||
h('details', { class: 'scraper-completed', 'data-scraper-disclosure': 'completed', open: disclosureOpen.completed }, h('summary', {}, `Completed downloads · ${completed.length}`),
|
||||
h('p', { class: 'muted small' }, 'Clearing removes finished queue rows only. Library chapters and activity records stay.'),
|
||||
h('button', { class: 'btn small', disabled: !completed.length, onclick: async () => {
|
||||
if (confirm(`Clear ${completed.length} completed download job${completed.length === 1 ? '' : 's'}${sourceFilter ? ` from ${sourceFilter}` : ''} from the queue? Library chapters will stay.`)) {
|
||||
await command({ command: 'clear-completed', source: sourceFilter });
|
||||
}
|
||||
} }, `Clear completed downloads (${completed.length})`),
|
||||
h('div', { class: 'scraper-queue' }, completed.slice().reverse().map(item => jobRow(item)))));
|
||||
const checkFor = (manga, source) => (state.updateChecks || []).find(item => item.mangaId === manga.id && item.source === source);
|
||||
const updateRows = (state.library || []).filter(manga => !onlyEnabled || manga.autoUpdate)
|
||||
.filter(manga => !sourceFilter || manga.sources?.some(source => source.adapter === sourceFilter))
|
||||
.sort((a, b) => a.title.localeCompare(b.title));
|
||||
const updatesPanel = h('div', { 'data-scraper-tab': 'updates', hidden: route.tab !== 'updates' },
|
||||
h('h2', {}, 'Updates'),
|
||||
h('div', { class: 'field' },
|
||||
state.libraryStatus?.error ? h('div', { class: 'small error' }, `Library index refresh failed: ${state.libraryStatus.error}`) : null,
|
||||
state.libraryStatus?.refreshing ? h('div', { class: 'muted small', role: 'status' },
|
||||
state.libraryStatus.ready ? 'Refreshing library index; showing the previous snapshot.' : 'Loading library index… Queue and rate controls are ready.') : null,
|
||||
h('div', { class: 'row' },
|
||||
h('label', { class: 'row' }, autoEnabled, 'Nightly discovery'),
|
||||
h('label', {}, h('small', {}, 'Local time'), autoTime),
|
||||
h('button', { class: 'btn', onclick: () => command({ command: 'settings', settings: {
|
||||
autoUpdateEnabled: autoEnabled.checked, autoUpdateTime: autoTime.value,
|
||||
} }) }, 'Save schedule'),
|
||||
h('button', { class: 'btn', onclick: () => command({ command: 'discover' }) }, 'Check now')),
|
||||
h('div', { class: 'muted small' }, state.nextDiscoveryAt ? `Next check: ${new Date(state.nextDiscoveryAt).toLocaleString()}` : 'Nightly discovery is disabled.'),
|
||||
h('label', { class: 'row small' }, h('input', { type: 'checkbox', checked: onlyEnabled,
|
||||
onchange: e => { onlyEnabled = e.target.checked; draw(lastState); } }), 'Only auto-updated manga'),
|
||||
h('div', { class: 'scraper-update-list' }, updateRows.map(manga => {
|
||||
const checks = (manga.sources || []).filter(source => source.adapter && (!sourceFilter || source.adapter === sourceFilter))
|
||||
.map(source => checkFor(manga, source.adapter)).filter(Boolean);
|
||||
const newest = key => checks.map(check => check[key]).filter(Boolean).sort().at(-1);
|
||||
const found = newest('lastNewContentAt'), downloaded = newest('lastDownloadedAt');
|
||||
return h('div', { class: 'scraper-update-row' },
|
||||
h('label', { class: 'row' }, h('input', { type: 'checkbox', checked: !!manga.autoUpdate,
|
||||
onchange: e => command({ command: 'auto-update', id: manga.id, enabled: e.target.checked }) }), h('b', {}, manga.title)),
|
||||
h('span', { class: 'muted small' }, `${manga.status} · ${manga.chapters} chapters`),
|
||||
h('span', {}, `Last checked: ${when(newest('lastCheckedAt'))}`),
|
||||
h('span', {}, `Last downloaded: ${when(downloaded)}`),
|
||||
found && (!downloaded || found > downloaded) ? h('span', { class: 'small saved' }, `New content found ${when(found)}; download pending`) : null);
|
||||
})),
|
||||
!updateRows.length ? h('p', { class: 'muted small' }, 'No manga match this filter.') : null,
|
||||
h('div', { class: 'muted small' }, 'Ongoing manga default to enabled. Completed manga default to disabled. A successful check updates “Last checked”; a published chapter updates “Last downloaded”.')),
|
||||
h('details', { class: 'scraper-history', 'data-scraper-disclosure': 'history', open: disclosureOpen.history }, h('summary', {}, 'Recent check activity'),
|
||||
h('div', { class: 'scraper-queue' }, (state.history || []).filter(item => item.status === 'checked' && (!sourceFilter || item.source === sourceFilter)).slice(0, 20).map(item =>
|
||||
h('div', { class: 'scraper-item small' }, `${item.title} · ${item.source} · checked ${when(item.time)} · no new chapters`)))));
|
||||
const sourcesPanel = h('div', { 'data-scraper-tab': 'sources', hidden: route.tab !== 'sources' },
|
||||
h('h2', {}, 'Sources'),
|
||||
(state.adapters || []).filter(adapter => !sourceFilter || adapter.id === sourceFilter).map(adapter => {
|
||||
const running = activeEntries.find(entry => entry.source === adapter.id);
|
||||
const waiting = state.queue.filter(item => item.source === adapter.id && ['queued', 'retry_wait', 'ready'].includes(item.status)).length;
|
||||
const problems = state.queue.filter(item => item.source === adapter.id && ['error', 'quarantined', 'unavailable'].includes(item.status)).length;
|
||||
return h('section', { class: 'scraper-source-card' }, h('h3', {}, adapter.label),
|
||||
h('p', { class: 'muted small' }, `${running ? 'Active' : state.sourcePaused?.[adapter.id] ? 'Paused by source protection' : 'Idle'} · ${waiting} waiting · ${problems} need attention`),
|
||||
sourceRateEditors.find(editor => editor.querySelector('b')?.textContent === adapter.label));
|
||||
}), h('p', { class: 'muted small' }, 'Each source runs sequentially with its own limits. New limits apply when that source starts or resumes a manga.'));
|
||||
card.append(activityPanel, queuePanel, updatesPanel, sourcesPanel);
|
||||
};
|
||||
|
||||
async function load(force = false) {
|
||||
if (!active) return;
|
||||
try {
|
||||
const state = await api('api/admin/scraper');
|
||||
lastState = state;
|
||||
const focused = card.contains(document.activeElement) && /^(INPUT|SELECT|TEXTAREA)$/.test(document.activeElement.tagName);
|
||||
// Replacing a live <details> during polling is disruptive on mobile:
|
||||
// the browser can deliver its toggle after the replacement. Leave the
|
||||
// disclosure's DOM alone until the user closes it or takes an action.
|
||||
const readingDisclosure = [...card.querySelectorAll('details[data-scraper-disclosure]')]
|
||||
.some(details => details.open);
|
||||
if (force || (!focused && !dirty && !readingDisclosure)) draw(state);
|
||||
}
|
||||
catch (error) { draw({ available: false, error: error.message }); }
|
||||
clearTimeout(timer);
|
||||
if (active) timer = setTimeout(load, document.hidden ? 15000 : 3000);
|
||||
}
|
||||
return {
|
||||
activate() { active = true; if (lastState && !dirty) draw(lastState); load(); },
|
||||
deactivate() { active = false; clearTimeout(timer); },
|
||||
tab() {
|
||||
card.querySelectorAll('[data-scraper-tab]').forEach(panel => { panel.hidden = panel.getAttribute('data-scraper-tab') !== route.tab; });
|
||||
card.querySelectorAll('.settings-tabs a').forEach(link => {
|
||||
if (link.getAttribute('href') === '#admin/scraper/' + route.tab) link.setAttribute('aria-current', 'page');
|
||||
else link.removeAttribute('aria-current');
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function adminCard(id) {
|
||||
const card = h('section', { class: 'card' });
|
||||
main.append(card);
|
||||
section(id).append(card);
|
||||
const draw = data => {
|
||||
const s = data.scan;
|
||||
const scanText = s.running ? 'Scanning…' : s.error ? 'Last scan failed: ' + s.error
|
||||
@@ -80,11 +395,14 @@
|
||||
const isAdmin = h('input', { type: 'checkbox' });
|
||||
const msg = h('span', { class: 'small' });
|
||||
card.textContent = '';
|
||||
card.append(h('h2', {}, 'Admin'),
|
||||
if (id === 'admin/library') card.append(h('h2', {}, 'Library maintenance'),
|
||||
h('div', { class: 'field' }, h('span', {}, 'Library'), h('div', { class: 'muted small' }, s.library), h('div', {}, scanText),
|
||||
h('div', { class: 'row', style: 'margin-top:8px' },
|
||||
h('button', { class: 'btn', onclick: rescan(false) }, 'Rescan'),
|
||||
h('button', { class: 'btn', onclick: rescan(true), title: 'Also re-read all image sizes' }, 'Full rescan'))),
|
||||
h('p', { class: 'muted small' }, 'Deployment, mounts and service startup are managed by Docker Compose. Library scans do not change download settings.'),
|
||||
h('a', { class: 'btn', href: 'health.html' }, 'Service health & incidents'));
|
||||
if (id === 'admin/users') card.append(h('h2', {}, 'User administration'),
|
||||
h('div', { class: 'field' }, h('span', {}, 'Users'),
|
||||
h('div', { class: 'users' }, data.users.map(u => h('div', {},
|
||||
h('span', {}, u.name, u.admin ? h('span', { class: 'tag', style: 'margin-left:8px' }, 'admin') : null),
|
||||
@@ -99,7 +417,36 @@
|
||||
catch (e) { msg.textContent = e.message; msg.className = 'small error'; }
|
||||
} }, 'Save user'), msg)));
|
||||
};
|
||||
async function load() { draw(await api('api/admin')); }
|
||||
load();
|
||||
async function load() {
|
||||
try { draw(await api('api/admin')); }
|
||||
catch (e) { card.textContent = ''; card.append(h('p', { class: 'error', role: 'alert' }, e.message), h('button', { class: 'btn', onclick: load }, 'Try again')); }
|
||||
}
|
||||
return { activate: load, deactivate() {} };
|
||||
}
|
||||
|
||||
function navigate(focus = false) {
|
||||
const next = settingsRoute(location.hash, me.admin);
|
||||
const entry = SETTINGS_SECTIONS.find(item => item.id === next.id);
|
||||
if (entry.href) { location.href = entry.href; return; }
|
||||
if (route.id !== next.id) controllers.get(route.id)?.deactivate();
|
||||
const changed = route.id !== next.id;
|
||||
route = next;
|
||||
shell.select(route.id, focus);
|
||||
if (!controllers.has(route.id) && route.id.startsWith('admin/')) {
|
||||
controllers.set(route.id, route.id === 'admin/scraper' ? scraperCard() : adminCard(route.id));
|
||||
}
|
||||
shell.content.replaceChildren(section(route.id));
|
||||
const controller = controllers.get(route.id);
|
||||
if (changed || !controller?.started) {
|
||||
if (controller) { controller.started = true; controller.activate(); }
|
||||
} else controller?.tab?.();
|
||||
// Normalize unknown/unauthorized routes without creating an extra history entry.
|
||||
if (location.hash !== '#' + route.id && !location.hash.startsWith('#' + route.id + '/')) {
|
||||
history.replaceState(null, '', '#' + route.id);
|
||||
}
|
||||
}
|
||||
window.addEventListener('hashchange', () => navigate(true));
|
||||
window.addEventListener('pagehide', () => controllers.get(route.id)?.deactivate());
|
||||
window.addEventListener('pageshow', e => { if (e.persisted) controllers.get(route.id)?.activate(); });
|
||||
navigate();
|
||||
})();
|
||||
@@ -0,0 +1,300 @@
|
||||
'use strict';
|
||||
|
||||
const INPUT_TYPES = ['mouse', 'touch', 'keyboard'];
|
||||
const MAX_RECENT_SESSIONS = 8;
|
||||
|
||||
const whole = (value, max = Number.MAX_SAFE_INTEGER) => Math.min(max, Math.max(0, Math.round(Number(value) || 0)));
|
||||
|
||||
function rangeSet(ranges, pageCount = Number.MAX_SAFE_INTEGER) {
|
||||
const out = new Set();
|
||||
for (const pair of Array.isArray(ranges) ? ranges : []) {
|
||||
if (!Array.isArray(pair) || pair.length < 2) continue;
|
||||
const start = whole(pair[0], pageCount - 1), end = whole(pair[1], pageCount - 1);
|
||||
for (let i = Math.min(start, end); i <= Math.max(start, end); i++) out.add(i);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function compactRanges(values) {
|
||||
const sorted = [...new Set(values)].filter(Number.isInteger).sort((a, b) => a - b);
|
||||
const ranges = [];
|
||||
for (const value of sorted) {
|
||||
const last = ranges[ranges.length - 1];
|
||||
if (last && value <= last[1] + 1) last[1] = Math.max(last[1], value);
|
||||
else ranges.push([value, value]);
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
function eligiblePages(version, settings) {
|
||||
const eligible = [];
|
||||
for (let i = 0; i < version.pages.length; i++) {
|
||||
const tags = version.pages[i].t || [];
|
||||
if (settings.skipExtras && tags.includes('extra')) continue;
|
||||
if (settings.skipCovers && tags.includes('cover')) continue;
|
||||
eligible.push(i);
|
||||
}
|
||||
return eligible.length ? eligible : version.pages.map((_, i) => i);
|
||||
}
|
||||
|
||||
function emptyRecord() {
|
||||
return {
|
||||
schema: 1,
|
||||
totalActiveMs: 0,
|
||||
firstCompleteMs: null,
|
||||
lastCompleteMs: null,
|
||||
loadingMs: 0,
|
||||
sessions: 0,
|
||||
completedPasses: 0,
|
||||
inputMs: { mouse: 0, touch: 0, keyboard: 0 },
|
||||
fastBursts: 0,
|
||||
rejectedPageViews: 0,
|
||||
maxObservedPps: 0,
|
||||
currentPass: null,
|
||||
recentSessions: [],
|
||||
updatedAt: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeRecord(record) {
|
||||
const base = emptyRecord(), out = record && typeof record === 'object' ? record : {};
|
||||
for (const [key, value] of Object.entries(base)) if (out[key] === undefined) out[key] = value;
|
||||
out.inputMs = { ...base.inputMs, ...(out.inputMs || {}) };
|
||||
out.recentSessions = Array.isArray(out.recentSessions) ? out.recentSessions : [];
|
||||
return out;
|
||||
}
|
||||
|
||||
function publicSummary(record, version, eligible) {
|
||||
const r = normalizeRecord(record || emptyRecord());
|
||||
const pass = r.currentPass && r.currentPass.version === version ? r.currentPass : null;
|
||||
const allowed = new Set(eligible);
|
||||
const seen = pass ? [...rangeSet(pass.seenRanges)].filter(x => allowed.has(x)) : [];
|
||||
const predominantInput = INPUT_TYPES.reduce((best, key) => r.inputMs[key] > r.inputMs[best] ? key : best, 'mouse');
|
||||
const hasInput = INPUT_TYPES.some(key => r.inputMs[key] > 0);
|
||||
return {
|
||||
schema: 1,
|
||||
totalActiveMs: whole(r.totalActiveMs),
|
||||
firstCompleteMs: r.firstCompleteMs == null ? null : whole(r.firstCompleteMs),
|
||||
lastCompleteMs: r.lastCompleteMs == null ? null : whole(r.lastCompleteMs),
|
||||
loadingMs: whole(r.loadingMs),
|
||||
sessions: whole(r.sessions),
|
||||
completedPasses: whole(r.completedPasses),
|
||||
inputMs: Object.fromEntries(INPUT_TYPES.map(key => [key, whole(r.inputMs[key])])),
|
||||
predominantInput: hasInput ? predominantInput : null,
|
||||
fastBursts: whole(r.fastBursts),
|
||||
rejectedPageViews: whole(r.rejectedPageViews),
|
||||
maxObservedPps: whole(r.maxObservedPps),
|
||||
eligiblePages: eligible.length,
|
||||
currentPass: {
|
||||
version,
|
||||
activeMs: whole(pass && pass.activeMs),
|
||||
seenRanges: compactRanges(seen),
|
||||
seenPages: seen.length,
|
||||
complete: !!(pass && pass.completedAt),
|
||||
startedAt: pass ? whole(pass.startedAt) : null,
|
||||
protocol: pass?.protocol || 1,
|
||||
requiredPages: Math.ceil(eligible.length * (pass?.protocol === 2 ? 0.6 : 1)),
|
||||
},
|
||||
updatedAt: whole(r.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function chapterInsight(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
const r = normalizeRecord(record);
|
||||
const totalActiveMs = whole(r.totalActiveMs);
|
||||
if (!totalActiveMs && !whole(r.completedPasses) && (!whole(r.updatedAt) || r.currentPass?.protocol === 2)) return null;
|
||||
return {
|
||||
totalActiveMs,
|
||||
firstCompleteMs: r.firstCompleteMs == null ? null : whole(r.firstCompleteMs),
|
||||
lastCompleteMs: r.lastCompleteMs == null ? null : whole(r.lastCompleteMs),
|
||||
completedPasses: whole(r.completedPasses),
|
||||
currentPassMs: whole(r.currentPass && r.currentPass.activeMs),
|
||||
currentPassComplete: !!(r.currentPass && r.currentPass.completedAt),
|
||||
updatedAt: whole(r.qualifiedUpdatedAt || r.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function median(values) {
|
||||
const sorted = values.filter(value => Number.isFinite(value) && value > 0).sort((a, b) => a - b);
|
||||
if (!sorted.length) return null;
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 ? whole(sorted[middle]) : whole((sorted[middle - 1] + sorted[middle]) / 2);
|
||||
}
|
||||
|
||||
function mangaInsight(records, chapterIds) {
|
||||
const source = records && typeof records === 'object' ? records : {};
|
||||
const ids = Array.isArray(chapterIds) ? chapterIds : Object.keys(source);
|
||||
const chapters = ids.map(id => ({ id, reading: chapterInsight(source[id]) })).filter(item => item.reading);
|
||||
const latest = chapters.reduce((best, item) => !best || item.reading.updatedAt > best.reading.updatedAt ? item : best, null);
|
||||
const completed = chapters.filter(item => item.reading.completedPasses > 0);
|
||||
return {
|
||||
activeMs: chapters.reduce((sum, item) => sum + item.reading.totalActiveMs, 0),
|
||||
timedChapters: chapters.filter(item => item.reading.totalActiveMs > 0).length,
|
||||
completedChapters: completed.length,
|
||||
completedPasses: completed.reduce((sum, item) => sum + item.reading.completedPasses, 0),
|
||||
rereads: completed.reduce((sum, item) => sum + Math.max(0, item.reading.completedPasses - 1), 0),
|
||||
typicalChapterMs: median(completed.map(item => item.reading.lastCompleteMs)),
|
||||
lastChapterId: latest ? latest.id : null,
|
||||
updatedAt: latest ? latest.reading.updatedAt : 0,
|
||||
};
|
||||
}
|
||||
|
||||
function libraryInsight(mangas) {
|
||||
const tracked = (Array.isArray(mangas) ? mangas : []).filter(item => item && item.reading && item.reading.activeMs > 0);
|
||||
const most = tracked.reduce((best, item) => !best || item.reading.activeMs > best.reading.activeMs ? item : best, null);
|
||||
const latest = tracked.reduce((best, item) => !best || item.reading.updatedAt > best.reading.updatedAt ? item : best, null);
|
||||
return {
|
||||
activeMs: tracked.reduce((sum, item) => sum + item.reading.activeMs, 0),
|
||||
timedChapters: tracked.reduce((sum, item) => sum + item.reading.timedChapters, 0),
|
||||
completedPasses: tracked.reduce((sum, item) => sum + item.reading.completedPasses, 0),
|
||||
rereads: tracked.reduce((sum, item) => sum + item.reading.rereads, 0),
|
||||
timedMangas: tracked.length,
|
||||
mostTimeMangaId: most ? most.id : null,
|
||||
lastMangaId: latest ? latest.id : null,
|
||||
lastChapterId: latest ? latest.reading.lastChapterId : null,
|
||||
updatedAt: latest ? latest.reading.updatedAt : 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Cumulative visits tolerate retry and an exit beacon overtaking a regular pulse.
|
||||
// Coverage belongs to THIS visit, so an old read cannot qualify a picture lookup.
|
||||
// Legacy totals are preserved; unqualified visit time is a draft, not a statistic.
|
||||
function applyVisit(record, pulse, { version, pageCount, eligible, now = Date.now() } = {}) {
|
||||
const r = normalizeRecord(record), session = String(pulse.session || ''), seq = whole(pulse.seq, 1e9);
|
||||
let receipt = r.recentSessions.find(item => item.id === session);
|
||||
if (receipt && seq <= whole(receipt.seq)) return { duplicate: true, record: r };
|
||||
if (!receipt) {
|
||||
if (!r.qualifiedUpdatedAt && r.totalActiveMs) r.qualifiedUpdatedAt = r.updatedAt;
|
||||
receipt = { id: session, seq: 0, at: now, pass: {
|
||||
protocol: 2, visit: session, version, pageCount, seenRanges: [], activeMs: 0,
|
||||
loadingMs: 0, inputMs: { mouse: 0, touch: 0, keyboard: 0 }, creditedMs: 0,
|
||||
startedAt: now, completedAt: null,
|
||||
} };
|
||||
r.recentSessions.push(receipt);
|
||||
r.currentPass = receipt.pass;
|
||||
}
|
||||
const pass = receipt.pass;
|
||||
if (!pass || pass.version !== version) return { duplicate: true, record: r };
|
||||
// Wall-time sanity, not a speed test or the legacy twelve-second packet cap.
|
||||
const budget = whole(pulse.elapsedMs, 24 * 60 * 60 * 1000) + 1500;
|
||||
const requestedActive = whole(pulse.activeMs, budget), requestedLoading = whole(pulse.loadingMs, budget);
|
||||
const scale = requestedActive + requestedLoading > budget ? budget / (requestedActive + requestedLoading) : 1;
|
||||
const active = Math.max(pass.activeMs, Math.round(requestedActive * scale));
|
||||
const loading = Math.max(pass.loadingMs, Math.round(requestedLoading * scale));
|
||||
r.loadingMs += loading - pass.loadingMs;
|
||||
pass.activeMs = active; pass.loadingMs = loading;
|
||||
const input = Object.fromEntries(INPUT_TYPES.map(key => [key, whole(pulse.inputMs?.[key], active)]));
|
||||
const inputTotal = INPUT_TYPES.reduce((sum, key) => sum + input[key], 0);
|
||||
const inputScale = inputTotal > active ? active / inputTotal : 1;
|
||||
for (const key of INPUT_TYPES) pass.inputMs[key] = Math.max(pass.inputMs[key], Math.round(input[key] * inputScale));
|
||||
const allowed = new Set(eligible), seen = rangeSet(pass.seenRanges, pageCount);
|
||||
for (const page of Array.isArray(pulse.seen) ? pulse.seen.slice(0, pageCount) : []) {
|
||||
if (Number.isInteger(page) && allowed.has(page)) seen.add(page);
|
||||
}
|
||||
pass.seenRanges = compactRanges(seen);
|
||||
const covered = eligible.filter(page => seen.has(page)).length;
|
||||
if (!pass.completedAt && eligible.length && covered >= Math.ceil(eligible.length * 0.6) && active > 0) {
|
||||
pass.completedAt = now;
|
||||
pass.ordinal = ++r.completedPasses;
|
||||
r.sessions++;
|
||||
}
|
||||
if (pass.completedAt) {
|
||||
r.totalActiveMs += active - pass.creditedMs;
|
||||
pass.creditedMs = active;
|
||||
pass.creditedInput = pass.creditedInput || { mouse: 0, touch: 0, keyboard: 0 };
|
||||
for (const key of INPUT_TYPES) {
|
||||
r.inputMs[key] += pass.inputMs[key] - pass.creditedInput[key];
|
||||
pass.creditedInput[key] = pass.inputMs[key];
|
||||
}
|
||||
// Duration keeps growing through the last page and departure, not just 60%.
|
||||
if (pass.ordinal === 1) r.firstCompleteMs = active;
|
||||
if (pass.ordinal === r.completedPasses) r.lastCompleteMs = active;
|
||||
r.qualifiedUpdatedAt = now;
|
||||
}
|
||||
pass.closed = !!pulse.final;
|
||||
receipt.seq = seq; receipt.at = now;
|
||||
if (r.currentPass?.visit === session) r.currentPass = pass;
|
||||
r.recentSessions.sort((a, b) => whole(b.at) - whole(a.at));
|
||||
r.recentSessions.splice(MAX_RECENT_SESSIONS);
|
||||
r.updatedAt = now;
|
||||
return { duplicate: false, record: r };
|
||||
}
|
||||
|
||||
function applyPulse(record, pulse, { version, pageCount, eligible, now = Date.now(), maxDeltaMs = 12000 } = {}) {
|
||||
const r = normalizeRecord(record);
|
||||
const session = String(pulse.session || '');
|
||||
const seq = whole(pulse.seq, 1e9);
|
||||
let receipt = r.recentSessions.find(item => item.id === session);
|
||||
if (receipt && seq <= whole(receipt.seq, 1e9)) return { duplicate: true, record: r };
|
||||
|
||||
const newSession = !receipt;
|
||||
if (!receipt) {
|
||||
receipt = { id: session, seq: 0, at: now, counted: false };
|
||||
r.recentSessions.push(receipt);
|
||||
}
|
||||
|
||||
const elapsed = newSession ? maxDeltaMs : Math.max(0, now - whole(receipt.at));
|
||||
const allowedDelta = Math.min(maxDeltaMs, elapsed + 1500);
|
||||
const requestedActive = whole(pulse.activeMs, maxDeltaMs * 2);
|
||||
const requestedLoading = whole(pulse.loadingMs, maxDeltaMs * 2);
|
||||
const requestedTotal = requestedActive + requestedLoading;
|
||||
const scale = requestedTotal > allowedDelta && requestedTotal ? allowedDelta / requestedTotal : 1;
|
||||
const activeMs = Math.round(requestedActive * scale);
|
||||
const loadingMs = Math.round(requestedLoading * scale);
|
||||
|
||||
receipt.seq = seq;
|
||||
receipt.at = now;
|
||||
if (!receipt.counted && activeMs > 0) { receipt.counted = true; r.sessions++; }
|
||||
r.recentSessions.sort((a, b) => whole(b.at) - whole(a.at));
|
||||
r.recentSessions.splice(MAX_RECENT_SESSIONS);
|
||||
|
||||
const prior = r.currentPass;
|
||||
if (!prior || prior.version !== version || (prior.completedAt && newSession)) {
|
||||
r.currentPass = {
|
||||
version,
|
||||
pageCount,
|
||||
seenRanges: [],
|
||||
activeMs: 0,
|
||||
startedAt: now,
|
||||
completedAt: null,
|
||||
};
|
||||
}
|
||||
const pass = r.currentPass;
|
||||
pass.pageCount = pageCount;
|
||||
pass.activeMs = whole(pass.activeMs) + activeMs;
|
||||
|
||||
r.totalActiveMs = whole(r.totalActiveMs) + activeMs;
|
||||
r.loadingMs = whole(r.loadingMs) + loadingMs;
|
||||
const rawInput = pulse.inputMs && typeof pulse.inputMs === 'object' ? pulse.inputMs : {};
|
||||
const requestedInput = Object.fromEntries(INPUT_TYPES.map(key => [key, whole(rawInput[key], maxDeltaMs * 2)]));
|
||||
const inputTotal = INPUT_TYPES.reduce((sum, key) => sum + requestedInput[key], 0);
|
||||
const inputScale = inputTotal > activeMs && inputTotal ? activeMs / inputTotal : 1;
|
||||
for (const key of INPUT_TYPES) r.inputMs[key] = whole(r.inputMs[key]) + Math.round(requestedInput[key] * inputScale);
|
||||
|
||||
const allowedPages = new Set(eligible);
|
||||
const seen = rangeSet(pass.seenRanges, pageCount);
|
||||
for (const raw of Array.isArray(pulse.seen) ? pulse.seen.slice(0, pageCount) : []) {
|
||||
const page = Number(raw);
|
||||
if (Number.isInteger(page) && allowedPages.has(page)) seen.add(page);
|
||||
}
|
||||
pass.seenRanges = compactRanges([...seen]);
|
||||
|
||||
r.fastBursts = whole(r.fastBursts) + whole(pulse.fastBursts, 100);
|
||||
r.rejectedPageViews = whole(r.rejectedPageViews) + whole(pulse.rejectedPageViews, pageCount * 2);
|
||||
r.maxObservedPps = Math.max(whole(r.maxObservedPps), whole(pulse.maxPps, 1000));
|
||||
|
||||
if (!pass.completedAt && eligible.length && eligible.every(page => seen.has(page))) {
|
||||
pass.completedAt = now;
|
||||
r.completedPasses = whole(r.completedPasses) + 1;
|
||||
r.lastCompleteMs = pass.activeMs;
|
||||
if (r.firstCompleteMs == null) r.firstCompleteMs = pass.activeMs;
|
||||
}
|
||||
r.updatedAt = now;
|
||||
return { duplicate: false, record: r, activeMs, loadingMs };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
INPUT_TYPES, applyPulse, applyVisit, chapterInsight, compactRanges, eligiblePages, emptyRecord,
|
||||
libraryInsight, mangaInsight, publicSummary, rangeSet,
|
||||
};
|
||||
@@ -0,0 +1,71 @@
|
||||
'use strict';
|
||||
|
||||
// Shared by the scanner and publisher. Dimensions identify strip layout, not
|
||||
// language or the reading order of a printed page. RTL is only a default.
|
||||
const KEY = 'x-reading-direction';
|
||||
const aliases = { rtl: 'rtl', ltr: 'ltr', ttb: 'ttb', vertical: 'ttb', webtoon: 'ttb' };
|
||||
const normalizeDirection = value => aliases[String(value || '').trim().toLowerCase()] || null;
|
||||
|
||||
function manualDirection(meta = {}) {
|
||||
const override = normalizeDirection(meta.directionOverride);
|
||||
if (override) return override;
|
||||
const value = normalizeDirection(meta.direction);
|
||||
if (!value) return null;
|
||||
const record = meta[KEY];
|
||||
// Legacy plain directions are ambiguous: never assume a human didn't set one.
|
||||
// Editing a generated value without its provenance is also a manual override.
|
||||
if (!record || record.version !== 1 || record.value !== value ||
|
||||
!['detected', 'fallback'].includes(record.source)) return value;
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectVerticalStrip(pages = []) {
|
||||
const dimensions = pages.map(p => {
|
||||
const rotated = p.r === 90 || p.r === 270;
|
||||
return { w: Number(rotated ? p.h : p.w), h: Number(rotated ? p.w : p.h) };
|
||||
}).filter(p => Number.isFinite(p.w) && Number.isFinite(p.h) && p.w > 0 && p.h > 0);
|
||||
const tall = dimensions.filter(p => p.h / p.w >= 4);
|
||||
const widths = tall.map(p => p.w).sort((a, b) => a - b);
|
||||
const middle = widths.length ? widths[Math.floor(widths.length / 2)] : 0;
|
||||
const consistent = tall.filter(p => Math.abs(p.w - middle) <= middle * .25).length;
|
||||
const evidence = { pages: pages.length, knownPages: dimensions.length, tallPages: tall.length,
|
||||
consistentTallPages: consistent, medianStripWidth: middle };
|
||||
// At least three coherent strips, >=75% of known pages, and >=75% known
|
||||
// dimensions. A landscape credits card may coexist with the vertical body.
|
||||
const detected = tall.length >= 3 && dimensions.length >= pages.length * .75 &&
|
||||
tall.length >= dimensions.length * .75 && consistent >= tall.length * .75;
|
||||
return detected ? { value: 'ttb', source: 'detected', evidence } : null;
|
||||
}
|
||||
|
||||
function fallbackDirection() {
|
||||
return { direction: 'rtl', [KEY]: { version: 1, value: 'rtl', source: 'fallback' } };
|
||||
}
|
||||
|
||||
// Scopes are chapter -> book -> manga (or existing -> incoming when publishing).
|
||||
// Any explicit choice beats inference; a more specific explicit choice wins.
|
||||
function resolveReadingDirection(scopes = [], pages = []) {
|
||||
for (let i = 0; i < scopes.length; i++) {
|
||||
const value = manualDirection(scopes[i]);
|
||||
if (value) return { value, source: 'manual', scope: i };
|
||||
}
|
||||
const detected = detectVerticalStrip(pages);
|
||||
if (detected) return detected;
|
||||
for (let i = 0; i < scopes.length; i++) {
|
||||
const meta = scopes[i] || {}, record = meta[KEY];
|
||||
if (record?.version === 1 && record.source === 'detected' &&
|
||||
record.value === normalizeDirection(meta.direction) && record.value === 'ttb') {
|
||||
return { value: 'ttb', source: 'detected', scope: i, evidence: record.evidence };
|
||||
}
|
||||
}
|
||||
return { value: 'rtl', source: 'fallback' };
|
||||
}
|
||||
|
||||
function directionFields(scopes, pages) {
|
||||
const { value, source, evidence } = resolveReadingDirection(scopes, pages);
|
||||
return { direction: value,
|
||||
...(source === 'manual' && scopes.some(m => normalizeDirection(m?.directionOverride)) ? { directionOverride: value } : {}),
|
||||
[KEY]: { version: 1, value, source, ...(evidence ? { evidence } : {}) } };
|
||||
}
|
||||
|
||||
module.exports = { KEY, normalizeDirection, manualDirection, detectVerticalStrip,
|
||||
fallbackDirection, resolveReadingDirection, directionFields };
|
||||
@@ -0,0 +1,7 @@
|
||||
'use strict';
|
||||
const fs = require('fs');
|
||||
function secret(name) {
|
||||
if (process.env[`${name}_FILE`]) return fs.readFileSync(process.env[`${name}_FILE`], 'utf8').trim();
|
||||
return process.env[name] || '';
|
||||
}
|
||||
module.exports = { secret };
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
const { Supervisor } = require('./supervisor');
|
||||
const supervisor = new Supervisor(process.argv[2] || 'reader');
|
||||
supervisor.start();
|
||||
const keepalive = setInterval(() => {}, 60000);
|
||||
let closing = false;
|
||||
for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, async () => {
|
||||
if (closing) return; closing = true; clearInterval(keepalive); await supervisor.close(); process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
'use strict';
|
||||
const fs = require('fs'), path = require('path'), { spawn } = require('child_process');
|
||||
const SPECS = {
|
||||
reader: { script: 'server.js', port: () => process.env.PORT || 8080, file: () => path.join(process.env.DATA || 'data', 'runtime-reader.json') },
|
||||
scraper: { script: 'scraper/daemon.js', port: () => process.env.SCRAPER_PORT || 8091, file: () => path.join(process.env.SCRAPER_DATA || path.join(process.env.DATA || 'data', 'scraper'), 'runtime.json') },
|
||||
sentinel: { script: 'sentinel/daemon.js', port: () => process.env.SENTINEL_PORT || 8092, file: () => path.join(process.env.SENTINEL_DATA || path.join(process.env.DATA || 'data', 'sentinel'), 'runtime.json') },
|
||||
};
|
||||
class Supervisor {
|
||||
constructor(name, options = {}) {
|
||||
this.name = name; this.spec = SPECS[name]; if (!this.spec) throw new Error('Unknown service');
|
||||
this.root = options.root || path.resolve(__dirname, '..'); this.child = null; this.stopped = false; this.timer = null;
|
||||
this.healthFailures = 0; this.restarting = false; this.startedAt = 0;
|
||||
this.spawn = options.spawn || spawn; this.retryBaseMs = options.retryBaseMs ?? 5000;
|
||||
this.file = options.file || path.resolve(this.root, this.spec.file()); fs.mkdirSync(path.dirname(this.file), { recursive: true });
|
||||
this.state = { failures: [], status: 'starting' };
|
||||
if (fs.existsSync(this.file)) this.state = JSON.parse(fs.readFileSync(this.file, 'utf8'));
|
||||
}
|
||||
save() { fs.writeFileSync(this.file + '.tmp', JSON.stringify(this.state, null, 2) + '\n'); fs.renameSync(this.file + '.tmp', this.file); }
|
||||
log(event, extra = {}) { console.log(JSON.stringify({ service: 'supervisor', target: this.name, event, ...extra })); }
|
||||
start() {
|
||||
if (this.stopped) return;
|
||||
this.state.failures = this.state.failures.filter(at => Date.now() - at < 3600000);
|
||||
if (this.state.failures.length >= 3) { this.state.status = 'parked'; this.save(); this.log('repair-required', { reason: 'Three failures within one hour' }); return; }
|
||||
this.child = this.spawn(process.execPath, [this.spec.script], { cwd: this.root, env: process.env, stdio: 'inherit', windowsHide: true });
|
||||
this.state.status = 'running'; this.state.pid = this.child.pid; this.startedAt = Date.now(); this.healthFailures = 0; this.restarting = false; this.save();
|
||||
this.log('started', { pid: this.child.pid });
|
||||
const child = this.child;
|
||||
child.on('error', e => this.log('spawn-error', { message: e.message }));
|
||||
child.once('close', (code, signal) => {
|
||||
if (this.child !== child) return;
|
||||
this.child = null; clearTimeout(this.killTimer);
|
||||
if (this.stopped) return;
|
||||
clearTimeout(this.timer);
|
||||
this.state.failures.push(Date.now()); this.state.status = 'retry_wait'; this.state.lastExit = { at: new Date().toISOString(), code, signal }; this.save();
|
||||
this.log('exited', { code, signal });
|
||||
this.timer = setTimeout(() => this.start(), Math.min(60000, this.retryBaseMs * 2 ** (this.state.failures.length - 1)));
|
||||
});
|
||||
this.timer = setTimeout(() => this.check(), 30000);
|
||||
}
|
||||
async check() {
|
||||
if (this.stopped || !this.child) return;
|
||||
const observedChild = this.child;
|
||||
const healthy = await fetch(`http://127.0.0.1:${this.spec.port()}/health`, { signal: AbortSignal.timeout(5000) }).then(async r => r.ok && (await r.json()).ok !== false).catch(() => false);
|
||||
if (this.stopped || this.child !== observedChild) return;
|
||||
this.healthFailures = healthy ? 0 : this.healthFailures + 1;
|
||||
if (this.healthFailures >= 3 && Date.now() - this.startedAt > 90000 && !this.restarting) {
|
||||
this.restarting = true; this.log('liveness-recovery');
|
||||
const child = this.child; child.kill('SIGTERM');
|
||||
this.killTimer = setTimeout(() => { if (this.child === child) child.kill('SIGKILL'); }, 65000);
|
||||
}
|
||||
if (!this.stopped) this.timer = setTimeout(() => this.check(), 30000);
|
||||
}
|
||||
async close() {
|
||||
this.stopped = true; clearTimeout(this.timer); clearTimeout(this.killTimer);
|
||||
if (!this.child) return;
|
||||
const child = this.child;
|
||||
await new Promise(resolve => {
|
||||
const timer = setTimeout(() => child.kill('SIGKILL'), 65000);
|
||||
child.once('exit', () => { clearTimeout(timer); resolve(); }); child.kill('SIGTERM');
|
||||
});
|
||||
this.state.status = 'stopped'; this.save();
|
||||
}
|
||||
}
|
||||
module.exports = { Supervisor, SPECS };
|
||||
@@ -0,0 +1,155 @@
|
||||
# MangaCircus scraper agent runbook
|
||||
|
||||
This document is normative for automated agents operating the scraper. Read it together with `../docs/scraper-guide.md`.
|
||||
|
||||
## 1. Non-negotiable safety policy
|
||||
|
||||
1. Scrape only works the operator is authorised to copy.
|
||||
2. Never bypass a CAPTCHA, Cloudflare challenge, login, paywall, geographic restriction, signed-URL check, or other access control.
|
||||
3. If a normal browser session displays a challenge, stop. A human may complete it normally; the agent may resume afterward. Repeated challenges end the run.
|
||||
4. Never request pages concurrently. Use the persisted per-host settings. The current operator-selected fast profile is 1–4 seconds between requests/images and 5–15 seconds between chapters. A host-specific published limit overrides these settings when it is slower.
|
||||
5. Stop immediately on HTTP 401, 403, or 429. On 5xx, retry at most twice after 30 and 60 seconds. Do not rotate IP addresses, identities, user agents, or mirrors to evade a block.
|
||||
6. Inspect metadata through AniList and its local 30-day cache. Do not revisit a reader/image host merely for authors, genres, descriptions, status, year, or cover metadata.
|
||||
7. Download only missing chapters. Never probe every historical chapter merely to discover the newest one when a chapter list is available.
|
||||
8. Publish only through the shared publisher, either `scraper/cli.js publish` or the
|
||||
managed daemon. Both validate images and use MangaCircus-safe staging and rename
|
||||
operations. Legacy conversion must be paused, validate every completed chapter,
|
||||
quarantine incomplete stages, and use an atomic manga-folder rename.
|
||||
|
||||
## 2. Pipeline
|
||||
|
||||
Runtime recovery is documented in `../docs/resilience.md`. Request-level 5xx retries
|
||||
remain limited to 30/60 seconds; after those are exhausted the daemon can resume the
|
||||
same checkpoint once after a five-minute source cooldown. No automatic access-error
|
||||
retry is permitted. Unavailable titles and parser failures are job-scoped; repeated
|
||||
independent parser failures stop the source lane. Sentinel never changes adapters.
|
||||
|
||||
```text
|
||||
public metadata -> cached series metadata
|
||||
chapter host -> newest chapter IDs only
|
||||
reader/browser -> ordered page manifest or local browser asset bundle
|
||||
manifest validation -> slow sequential acquisition -> hidden staging folder
|
||||
complete chapter -> one atomic rename -> optional one MangaCircus rescan per batch
|
||||
```
|
||||
|
||||
The interchange format is `manifest.example.json`. Each page has exactly one of:
|
||||
|
||||
- `path`: an already downloaded browser asset; preferred for protected/browser-only hosts.
|
||||
- `url`: a public image URL fetched sequentially by the publisher.
|
||||
|
||||
Do not store cookies, tokens, signed URLs, account identifiers, or other secrets in `manga.json`. Temporary manifests containing expiring URLs must stay outside the library and should be deleted after a successful import.
|
||||
|
||||
## 3. Metadata procedure
|
||||
|
||||
Run one AniList lookup per new series:
|
||||
|
||||
```bash
|
||||
node scraper/cli.js metadata-anilist "series title"
|
||||
```
|
||||
|
||||
Verify the result using title, native title, author and year. Copy the returned fields into `series.metadata`. Keep the chapter host URLs only in `series.sources`. If the lookup is ambiguous, stop for human selection; never silently attach metadata from a similarly named work.
|
||||
|
||||
For the selected Heavy Knight manga, the verified AniList identity is `150193` / MAL `147799`. AniList `150194` is the light novel and MUST NOT be used for manga metadata or cover art. For One Piece it is AniList `30013` / MAL `13`.
|
||||
|
||||
## 4. WeebCentral adapter
|
||||
|
||||
Observed 2026-10-01:
|
||||
|
||||
- Series: `https://weebcentral.com/series/<id>`
|
||||
- Full list: `/series/<id>/full-chapter-list`
|
||||
- Chapter: `/chapters/<id>`
|
||||
- Page fragment: `/chapters/<id>/images?is_prev=False&reading_style=long_strip¤t_page=1`
|
||||
|
||||
The fragment contains the ordered image elements. Fetch the series page once, the full list once, and only the selected chapter/page fragments. Current chapter 177 exposed 15 PNG pages. Do not fetch cover art from this host when AniList metadata is available.
|
||||
|
||||
```bash
|
||||
node scraper/cli.js discover-weebcentral "SERIES_URL" --limit 3 --out manifest.json
|
||||
node scraper/cli.js publish manifest.json --library library
|
||||
```
|
||||
|
||||
If the HTML no longer matches, stop and use a normal browser to inspect the visible reader. Do not guess endpoint grids.
|
||||
|
||||
## 5. ReiManga adapter
|
||||
|
||||
Observed 2026-10-01: anonymous scripted requests receive a managed Cloudflare challenge. The indexed series page lists chapters 177 downward, but that index is not sufficient to acquire pages.
|
||||
|
||||
Required approach:
|
||||
|
||||
1. Open the supplied series page in a normal browser.
|
||||
2. If a challenge appears, stop for the human. Do not automate it.
|
||||
3. After the user reaches the normal series page, open only the selected chapter.
|
||||
4. Read the visible chapter number/title/date and inventory the image assets already rendered by the page.
|
||||
5. Export only assets belonging to that chapter into a local temporary folder.
|
||||
6. Create a manifest with local `path` entries and source name `ReiManga`.
|
||||
7. Publish locally. Wait 30–60 seconds before opening another chapter.
|
||||
|
||||
If the site sends the chapter to another domain, record the ReiManga chapter URL as the source URL, but accept assets only from domains actually loaded by that reader page. Any login, paywall, error, or renewed challenge ends the run.
|
||||
|
||||
## 5A. MangaDex adapter
|
||||
|
||||
Observed 2026-10-03:
|
||||
|
||||
- Title: `https://mangadex.org/title/<uuid>`
|
||||
- API: `https://api.mangadex.org`
|
||||
- English feed: `/manga/<uuid>/feed`
|
||||
- Page manifest: `/at-home/server/<chapter-uuid>`
|
||||
- Images: the HTTPS MangaDex@Home host returned by that manifest
|
||||
|
||||
Only explicit MangaDex title UUIDs may enter the managed queue. Never automatically replace a queued source from fuzzy title search: verify the MangaDex AniList/MAL link against the selected work first. Select one stable release per numeric English chapter, credit MangaDex and the scanlation group in the chapter source, and ignore external-link or zero-page releases. Honor group removal requests. MangaDex-backed content must not be used to support advertising or paid access.
|
||||
|
||||
Unnumbered hosted releases are supported with `special-mangadex-<chapter-uuid>` keys
|
||||
and human labels, never guessed chapter numbers. Keep distinct source UUIDs separate.
|
||||
Use the queued catalog-only inspection and confirm the canonical edition before
|
||||
downloading a new title. Empty hosted-English catalogs distinguish external-only,
|
||||
no-English-listed and no-hosted-pages; none is an access error or a reason to probe
|
||||
external hosts. A finished job does not prove whole-series coverage. See
|
||||
`../docs/scraper-service.md` for inspection, scopes and availability semantics.
|
||||
|
||||
Use full-resolution `data` pages, validate that the manifest page count still matches the feed, and accept only HTTPS `uploads.mangadex.org` or `*.mangadex.network` image hosts. Keep one sequential MangaDex worker using its configured per-host rate (currently 1–4 seconds between requests and 5–15 seconds between chapters). Stop on 401, 403, or 429 exactly as for other sources; never add authentication or rotate hosts to evade a response.
|
||||
|
||||
## 6. MANGA Plus adapter
|
||||
|
||||
Observed 2026-10-01 for title `100020` (One Piece): the title page states that only the latest three chapters remain viewable. They are:
|
||||
|
||||
| Chapter | Viewer id | Published | Title |
|
||||
|---|---:|---|---|
|
||||
| 1192 | 1030379 | 2026-09-06 | We Won't Stand For It!!! |
|
||||
| 1193 | 1030380 | 2026-09-13 | Still Practicing |
|
||||
| 1194 | 1030381 | 2026-09-27 | All Things Are In Flux |
|
||||
|
||||
The normal web app calls `title_detailV4` and `manga_viewer_v3`, then renders signed `jumpg-assets*.tokyo-cdn.com` payloads. Direct non-browser API calls currently return an account-ban response. A browser asset export on 2026-10-01 produced payloads labelled `image/jpeg`, but their bytes were obfuscated and failed JPEG signature validation. Therefore:
|
||||
|
||||
1. Use only the normal MANGA Plus browser UI.
|
||||
2. Read the title page and select the last three chapters explicitly marked viewable.
|
||||
3. Open one viewer at a time, wait for its page count to stabilise, and inventory the rendered assets.
|
||||
4. Export only URLs matching `/secure/title/100020/chapter/<viewer-id>/manga_page/` from that page's observed assets.
|
||||
5. Validate the first exported payload by file signature before exporting or processing the rest. A `.jpg` name or HTTP content type is not proof of a JPEG.
|
||||
6. If valid images are available, preserve numeric order and use local `path` entries in the import manifest. Set source name `MANGA Plus`, source URL `https://mangaplus.shueisha.co.jp/viewer/<viewer-id>`, priority `1`, direction `rtl`.
|
||||
7. If payloads are obfuscated, abort the source. Do not decrypt them, extract pixels from the rendering canvas, or reconstruct chapters with automated screenshots.
|
||||
8. Wait at least 30 seconds between chapters. If the UI says a chapter is unavailable, do not attempt alternate APIs or URLs.
|
||||
|
||||
Do not implement API protobuf decoding, image deobfuscation, signed-URL generation, device-id synthesis, canvas extraction, screenshot reconstruction, or retry identity rotation. The browser-exported file boundary plus ordinary image validation is the allowed boundary. As of the observation date, chapters 1192–1194 are identified but **not importable by this compliant adapter**.
|
||||
|
||||
## 7. MangaCircus publication rules
|
||||
|
||||
- Use the flat book strategy: `v01`.
|
||||
- Chapter folders include stable source suffixes, for example `c1194 All Things Are In Flux [MANGA Plus]`.
|
||||
- Existing chapters are skipped by default. `--replace` replaces the entire version folder atomically; never overwrite image files in place.
|
||||
- New manga are built wholly under `.staging-mcscraper-*` and published with one rename.
|
||||
- The publisher sniffs image bytes; filename extensions are never trusted.
|
||||
- Page numbers are contiguous from `0001`.
|
||||
- Add `cover`, `double`, `extra`, or rotation tags only when certain. Do not infer tags from aspect ratio alone.
|
||||
- Trigger at most one MangaCircus rescan after the whole batch.
|
||||
|
||||
## 8. Abort checklist
|
||||
|
||||
Abort without workaround when any of these occurs:
|
||||
|
||||
- CAPTCHA, challenge, access-denied, rate-limit, account-ban, or login/paywall prompt.
|
||||
- Page count changes during acquisition.
|
||||
- Duplicate/non-numeric page order, unsupported image bytes, or an HTML error page disguised as an image.
|
||||
- Metadata match is ambiguous.
|
||||
- The target manga folder exists but belongs to another source identity.
|
||||
- A chapter would need to move books or rename an existing version folder.
|
||||
|
||||
Log the URL, status, and phase, but never log cookies, query signatures, passwords, or browser storage.
|
||||
@@ -0,0 +1,48 @@
|
||||
# MangaCircus scraper
|
||||
|
||||
The scraper uses small host adapters to create a common manifest, followed by one conservative publisher that writes the MangaCircus library atomically.
|
||||
|
||||
Start with [AGENT_RUNBOOK.md](AGENT_RUNBOOK.md). It contains the host-specific procedures, safety limits, and stop conditions. `manifest.example.json` documents the interchange format.
|
||||
|
||||
The historical human-readable wishlist is [WISHLIST.md](WISHLIST.md). Once the
|
||||
managed daemon has started, `data/scraper/queue.json` is the authoritative queue and
|
||||
is edited through the admin UI.
|
||||
|
||||
## Managed service
|
||||
|
||||
Run MangaCircus and the scraper daemon together:
|
||||
|
||||
```bash
|
||||
npm run start:all
|
||||
```
|
||||
|
||||
Or run them as two independently supervised processes:
|
||||
|
||||
```bash
|
||||
npm start
|
||||
npm run start:scraper
|
||||
```
|
||||
|
||||
The daemon binds to `127.0.0.1:8091` by default, keeps its durable queue in
|
||||
`data/scraper/queue.json`, and resumes incomplete archive checkpoints. An admin can
|
||||
control it from **Settings → Scraper**. Set `SCRAPER_TOKEN` on both processes when
|
||||
the daemon is not restricted to loopback.
|
||||
|
||||
Queue schema version 3 supports scoped jobs, multiple jobs for the same upstream
|
||||
series, one sequential worker per source, incremental chapter publication, library
|
||||
duplicate/version checks, and configurable nightly update discovery. See
|
||||
[`docs/scraper-service.md`](../docs/scraper-service.md) for the service contract and
|
||||
[`docs/local-deployment.md`](../docs/local-deployment.md) for the currently running
|
||||
Windows Scheduled Task deployment. On that machine the scheduled task owns the single
|
||||
supervisor; do not run `npm run start:all` beside it.
|
||||
|
||||
```bash
|
||||
npm run test:scraper
|
||||
node scraper/cli.js metadata-anilist "One Piece"
|
||||
node scraper/cli.js discover-weebcentral "https://weebcentral.com/series/..." --limit 3 --out manifest.json
|
||||
node scraper/cli.js publish manifest.json --library library --metadata-search "Series title"
|
||||
node scraper/cli.js archive-weebcentral "https://weebcentral.com/series/..." --library library --metadata-search "Series title"
|
||||
node scraper/cli.js status-weebcentral "https://weebcentral.com/series/..." --library library
|
||||
```
|
||||
|
||||
The implementation uses only Node.js built-ins. Browser-only sources must first export observed page assets to local files; the publisher consumes those files and does not automate access-control challenges.
|
||||
@@ -0,0 +1,95 @@
|
||||
'use strict';
|
||||
|
||||
const { runArchive } = require('../jobs/weebcentral-archive');
|
||||
const { runMangaDexArchive } = require('../jobs/mangadex-archive');
|
||||
const { SlowHttp } = require('../lib/http');
|
||||
const { discoverWeebCatalog } = require('./weebcentral');
|
||||
const { mangaDexInfo, discoverMangaDexCatalog } = require('./mangadex');
|
||||
const { ArchiveControlError } = require('../lib/control');
|
||||
|
||||
function controlledHttp(http, shouldStop) {
|
||||
return { async get(...args) {
|
||||
const check = () => { const reason = shouldStop?.(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); };
|
||||
check(); const response = await http.get(...args); check(); return response;
|
||||
} };
|
||||
}
|
||||
|
||||
function weebCentralInfo(value) {
|
||||
let url;
|
||||
try { url = new URL(String(value)); } catch { return null; }
|
||||
if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com') return null;
|
||||
const match = /^\/series\/([A-Z0-9]+)(?:\/([^/?#]+))?\/?$/i.exec(url.pathname);
|
||||
if (!match) return null;
|
||||
const title = decodeURIComponent(match[2] || match[1]).replace(/[-_]+/g, ' ').trim();
|
||||
return { source: 'weebcentral', id: match[1].toUpperCase(), title, url: `https://weebcentral.com/series/${match[1]}${match[2] ? `/${match[2]}` : ''}` };
|
||||
}
|
||||
|
||||
const adapters = [{
|
||||
id: 'weebcentral',
|
||||
label: 'WeebCentral',
|
||||
match: weebCentralInfo,
|
||||
async discover(item, options) {
|
||||
const http = new SlowHttp({
|
||||
minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2,
|
||||
onEvent: options.onEvent,
|
||||
});
|
||||
return discoverWeebCatalog(item.url, controlledHttp(http, options.shouldStop));
|
||||
},
|
||||
run(item, options) {
|
||||
return runArchive({
|
||||
seriesUrl: item.url,
|
||||
libraryRoot: options.libraryRoot,
|
||||
metadataSearch: item.metadataSearch || item.title,
|
||||
minMs: options.settings.minMs,
|
||||
maxMs: options.settings.maxMs,
|
||||
chapterMinMs: options.settings.chapterMinMs,
|
||||
chapterMaxMs: options.settings.chapterMaxMs,
|
||||
shouldStop: options.shouldStop,
|
||||
scope: item.scope,
|
||||
existingPolicy: item.existingPolicy,
|
||||
libraryMangaId: item.libraryMangaId,
|
||||
onPublished: options.onPublished,
|
||||
onCatalog: options.onCatalog,
|
||||
});
|
||||
},
|
||||
}, {
|
||||
id: 'mangadex',
|
||||
label: 'MangaDex',
|
||||
match: mangaDexInfo,
|
||||
async discover(item, options) {
|
||||
const http = new SlowHttp({
|
||||
minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2,
|
||||
userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)',
|
||||
onEvent: options.onEvent,
|
||||
});
|
||||
return discoverMangaDexCatalog(item.url, controlledHttp(http, options.shouldStop));
|
||||
},
|
||||
run(item, options) {
|
||||
return runMangaDexArchive({
|
||||
seriesUrl: item.url,
|
||||
title: item.title,
|
||||
libraryRoot: options.libraryRoot,
|
||||
metadataSearch: item.metadataSearch || item.title,
|
||||
minMs: options.settings.minMs,
|
||||
maxMs: options.settings.maxMs,
|
||||
chapterMinMs: options.settings.chapterMinMs,
|
||||
chapterMaxMs: options.settings.chapterMaxMs,
|
||||
shouldStop: options.shouldStop,
|
||||
scope: item.scope,
|
||||
existingPolicy: item.existingPolicy,
|
||||
libraryMangaId: item.libraryMangaId,
|
||||
onPublished: options.onPublished,
|
||||
onCatalog: options.onCatalog,
|
||||
});
|
||||
},
|
||||
}];
|
||||
|
||||
function adapterFor(url) {
|
||||
for (const adapter of adapters) {
|
||||
const info = adapter.match(url);
|
||||
if (info) return { adapter, info };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = { adapters, adapterFor, weebCentralInfo, mangaDexInfo };
|
||||
@@ -0,0 +1,161 @@
|
||||
'use strict';
|
||||
|
||||
const API = 'https://api.mangadex.org';
|
||||
const { specialKey } = require('../../chapter-identity');
|
||||
|
||||
function mangaDexInfo(value) {
|
||||
let url;
|
||||
try { url = new URL(String(value)); } catch { return null; }
|
||||
if (url.protocol !== 'https:' || !['mangadex.org', 'www.mangadex.org'].includes(url.hostname)) return null;
|
||||
const match = /^\/title\/([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})(?:\/([^/?#]+))?\/?$/i.exec(url.pathname);
|
||||
if (!match) return null;
|
||||
const id = match[1].toLowerCase();
|
||||
const title = decodeURIComponent(match[2] || id).replace(/[-_]+/g, ' ').trim();
|
||||
return { source: 'mangadex', id, title, url: `https://mangadex.org/title/${id}${match[2] ? `/${match[2]}` : ''}` };
|
||||
}
|
||||
|
||||
function localizedTitle(attributes = {}) {
|
||||
const titles = attributes.title || {};
|
||||
return titles.en || titles['ja-ro'] || titles.ja || Object.values(titles).find(Boolean) || 'Untitled';
|
||||
}
|
||||
|
||||
function chapterNumber(value) {
|
||||
const raw = String(value ?? '').trim();
|
||||
if (!/^\d+(?:\.\d+)?$/.test(raw)) return null;
|
||||
const [whole, decimal = ''] = raw.split('.');
|
||||
const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
|
||||
const normalizedDecimal = decimal.replace(/0+$/, '');
|
||||
return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
|
||||
}
|
||||
|
||||
function groupFor(resource) {
|
||||
const relation = (resource.relationships || []).find(item => item.type === 'scanlation_group');
|
||||
return relation ? { id: relation.id, name: relation.attributes?.name || 'Unknown group' } : null;
|
||||
}
|
||||
|
||||
function releaseRank(resource) {
|
||||
const group = groupFor(resource)?.name || '';
|
||||
const credited = /official|license/i.test(group) ? 0 : group ? 1 : 2;
|
||||
const date = Date.parse(resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt || '') || Number.MAX_SAFE_INTEGER;
|
||||
return [credited, date, resource.id];
|
||||
}
|
||||
|
||||
function compareRank(left, right) {
|
||||
const a = releaseRank(left), b = releaseRank(right);
|
||||
return a[0] - b[0] || a[1] - b[1] || String(a[2]).localeCompare(String(b[2]));
|
||||
}
|
||||
|
||||
function selectChapterReleases(resources = [], specialLabel = 'Special') {
|
||||
const selected = new Map();
|
||||
for (const resource of resources) {
|
||||
if (resource.attributes?.externalUrl || Number(resource.attributes?.pages || 0) < 1) continue;
|
||||
const number = chapterNumber(resource.attributes?.chapter) || specialKey(resource.id);
|
||||
const prior = selected.get(number);
|
||||
if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource);
|
||||
}
|
||||
return [...selected.entries()].map(([number, resource]) => {
|
||||
const group = groupFor(resource);
|
||||
return {
|
||||
id: resource.id, number, title: resource.attributes?.title || '',
|
||||
...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}),
|
||||
...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}),
|
||||
pages: Number(resource.attributes?.pages || 0),
|
||||
added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt,
|
||||
group,
|
||||
url: `https://mangadex.org/chapter/${resource.id}`,
|
||||
};
|
||||
}).sort((a, b) => Number(!!a.special) - Number(!!b.special) || (a.special ? a.number.localeCompare(b.number) : Number(a.number) - Number(b.number) || a.number.localeCompare(b.number)));
|
||||
}
|
||||
|
||||
function availabilitySummary(attributes, releases, chapters, externalReleases = null) {
|
||||
const numbered = chapters.filter(ref => !ref.special);
|
||||
const expected = chapterNumber(attributes.lastChapter);
|
||||
const maximum = numbered.length ? Math.max(...numbered.map(ref => Number(ref.number))) : null;
|
||||
const reason = chapters.length ? null : externalReleases > 0 ? 'external-only'
|
||||
: Array.isArray(attributes.availableTranslatedLanguages) && !attributes.availableTranslatedLanguages.includes('en') ? 'no-english' : 'no-hosted-pages';
|
||||
const message = reason === 'external-only' ? `MangaDex lists ${externalReleases} English external-link releases, but no hosted English pages. External links are not downloaded.`
|
||||
: reason === 'no-english' ? 'This MangaDex edition has no English releases listed.'
|
||||
: reason ? 'This MangaDex edition has no hosted English releases with pages.' : null;
|
||||
return { language: 'en', hostedReleases: releases.filter(r => !r.attributes?.externalUrl && Number(r.attributes?.pages) > 0).length,
|
||||
selectedChapters: chapters.length, numberedChapters: numbered.length, specialReleases: chapters.length - numbered.length,
|
||||
externalReleases, expectedLastChapter: expected, highestHostedChapter: maximum,
|
||||
coverage: expected && (maximum === null || maximum < Number(expected)) ? 'partial' : 'unknown', reason, message };
|
||||
}
|
||||
|
||||
async function jsonGet(http, url) {
|
||||
const response = await http.get(url, { accept: 'application/json' });
|
||||
const body = await response.json();
|
||||
if (body.result && body.result !== 'ok') throw new Error(`MangaDex API result: ${body.result}`);
|
||||
return body;
|
||||
}
|
||||
|
||||
async function discoverMangaDexCatalog(seriesUrl, http) {
|
||||
const info = mangaDexInfo(seriesUrl);
|
||||
if (!info) throw new Error('Invalid MangaDex title URL');
|
||||
const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data;
|
||||
if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
|
||||
|
||||
const releases = [];
|
||||
let offset = 0;
|
||||
for (;;) {
|
||||
const params = new URLSearchParams({ limit: '100', offset: String(offset), includeFutureUpdates: '0', includeExternalUrl: '0' });
|
||||
params.append('translatedLanguage[]', 'en');
|
||||
params.append('includes[]', 'scanlation_group');
|
||||
params.append('order[chapter]', 'asc');
|
||||
const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`);
|
||||
releases.push(...(page.data || []));
|
||||
offset += page.data?.length || 0;
|
||||
if (!page.data?.length || offset >= Number(page.total || 0)) break;
|
||||
}
|
||||
|
||||
const title = localizedTitle(manga.attributes);
|
||||
const specialLabel = /pre.serialization|pilot/i.test(title) ? 'Pilot'
|
||||
: (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
|
||||
const chapters = selectChapterReleases(releases, specialLabel);
|
||||
// External-only is a separate feed mode, not a combined hosted+external catalog.
|
||||
let externalReleases = null;
|
||||
if (!chapters.length) {
|
||||
const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' });
|
||||
params.append('translatedLanguage[]', 'en');
|
||||
externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0);
|
||||
}
|
||||
return {
|
||||
id: info.id,
|
||||
title,
|
||||
originalLanguage: manga.attributes?.originalLanguage || null,
|
||||
links: manga.attributes?.links || {},
|
||||
availability: availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases),
|
||||
chapters,
|
||||
};
|
||||
}
|
||||
|
||||
function pageUrls(atHome, chapterUrl, expectedPages) {
|
||||
const base = new URL(atHome.baseUrl);
|
||||
if (base.protocol !== 'https:' || !(base.hostname === 'uploads.mangadex.org' || base.hostname.endsWith('.mangadex.network'))) {
|
||||
throw new Error(`Unexpected MangaDex@Home host: ${base.hostname}`);
|
||||
}
|
||||
const hash = String(atHome.chapter?.hash || '');
|
||||
const files = atHome.chapter?.data;
|
||||
if (!hash || !Array.isArray(files) || !files.length) throw new Error('MangaDex@Home returned no pages');
|
||||
if (expectedPages && files.length !== expectedPages) throw new Error(`MangaDex page count changed: expected ${expectedPages}, received ${files.length}`);
|
||||
return files.map(file => ({
|
||||
url: new URL(`/data/${encodeURIComponent(hash)}/${encodeURIComponent(file)}`, base).href,
|
||||
headers: { referer: chapterUrl },
|
||||
}));
|
||||
}
|
||||
|
||||
async function discoverMangaDexChapter(ref, http) {
|
||||
const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`);
|
||||
const groupName = ref.group?.name ? ` · ${ref.group.name}` : '';
|
||||
return {
|
||||
number: ref.number, title: ref.title || '', added: ref.added,
|
||||
...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}),
|
||||
source: { name: `MangaDex${groupName}`, url: ref.url },
|
||||
pages: pageUrls(atHome, ref.url, ref.pages),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
API, mangaDexInfo, localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary,
|
||||
discoverMangaDexCatalog, discoverMangaDexChapter, pageUrls,
|
||||
};
|
||||
@@ -0,0 +1,88 @@
|
||||
'use strict';
|
||||
|
||||
const { SlowHttp } = require('../lib/http');
|
||||
const { fallbackDirection } = require('../../reading-direction');
|
||||
|
||||
const decode = value => value.replace(/&/g, '&').replace(/'/g, "'").replace(/"/g, '"').replace(/<[^>]+>/g, '').trim();
|
||||
|
||||
function absolute(base, value) { return new URL(value, base).href; }
|
||||
|
||||
function parseChapters(html, baseUrl) {
|
||||
const found = [];
|
||||
const re = /<a\b[^>]*href="([^"]*\/chapters\/([A-Z0-9]+)[^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
|
||||
for (const match of html.matchAll(re)) {
|
||||
const text = decode(match[3]).replace(/\s+/g, ' ');
|
||||
const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null;
|
||||
// A few WeebCentral catalogs use a series-specific label (for example
|
||||
// "Rating 442", "Episode 128", "Night 182", "Log. 65", or "Act 60.6") in the same place where most catalogs
|
||||
// say "Chapter 442". Treat those labels as chapter numbering instead of
|
||||
// discarding the entire catalog.
|
||||
const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1];
|
||||
const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1];
|
||||
const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null);
|
||||
if (number) found.push({
|
||||
id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(),
|
||||
...(season ? { book: { number: season, title: `Season ${season}` } } : {}),
|
||||
url: absolute(baseUrl, match[1]), raw: text,
|
||||
});
|
||||
}
|
||||
return [...new Map(found.map(chapter => [chapter.id, chapter])).values()];
|
||||
}
|
||||
|
||||
async function discoverWeebCatalog(seriesUrl, http) {
|
||||
const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
|
||||
const title = decode(seriesHtml.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled');
|
||||
const listUrl = seriesHtml.match(/hx-get="([^"]+\/full-chapter-list)"/i)?.[1];
|
||||
const listHtml = listUrl ? await (await http.get(absolute(seriesUrl, listUrl), { accept: 'text/html' })).text() : seriesHtml;
|
||||
const chapterLinks = [...listHtml.matchAll(/<a\b[^>]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length;
|
||||
const chapters = parseChapters(listHtml, seriesUrl).sort((a, b) =>
|
||||
Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number));
|
||||
return { title, chapters, chapterLinks,
|
||||
...(chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels',
|
||||
message: `WeebCentral lists ${chapterLinks} chapter links, but their numbering labels are not recognized by the parser.` } } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function discoverWeebChapter(chapter, http) {
|
||||
const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text();
|
||||
const manifestUrl = pageHtml.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1]
|
||||
|| `${chapter.url}/images?is_prev=False`;
|
||||
const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip¤t_page=1`, { accept: 'text/html' })).text();
|
||||
const pages = parsePageUrls(fragment, manifestUrl).map(url => ({ url, headers: { referer: chapter.url } }));
|
||||
if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`);
|
||||
return {
|
||||
number: chapter.number, title: chapter.title || '',
|
||||
...(chapter.book ? { book: chapter.book } : {}),
|
||||
added: pageHtml.match(/datetime="([^"]+)"/i)?.[1],
|
||||
source: { name: 'WeebCentral', url: chapter.url }, pages,
|
||||
};
|
||||
}
|
||||
|
||||
function parsePageUrls(html, baseUrl) {
|
||||
const urls = [];
|
||||
for (const match of html.matchAll(/<img\b[^>]*(?:src|data-src)="([^"]+)"/gi)) {
|
||||
const url = absolute(baseUrl, match[1].replace(/&/g, '&'));
|
||||
if (!url.startsWith('data:')) urls.push(url);
|
||||
}
|
||||
return [...new Set(urls)];
|
||||
}
|
||||
|
||||
async function discoverWeebCentral(seriesUrl, options = {}) {
|
||||
const http = new SlowHttp(options.http || {});
|
||||
const catalog = await discoverWeebCatalog(seriesUrl, http);
|
||||
const chapters = catalog.chapters
|
||||
.sort((a, b) => Number(b.number) - Number(a.number))
|
||||
.slice(0, options.limit || 3)
|
||||
.sort((a, b) => Number(a.number) - Number(b.number));
|
||||
const output = [];
|
||||
for (const chapter of chapters) {
|
||||
output.push(await discoverWeebChapter(chapter, http));
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
series: { title: catalog.title, ...fallbackDirection(), sources: [{ name: 'WeebCentral', url: seriesUrl }], scraper: { ids: { weebcentral: seriesUrl.split('/series/')[1]?.split('/')[0] } } },
|
||||
http: options.http || {}, chapters: output,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { discoverWeebCentral, discoverWeebCatalog, discoverWeebChapter, parseChapters, parsePageUrls };
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { publishManifest } = require('./lib/publisher');
|
||||
const { discoverWeebCentral } = require('./adapters/weebcentral');
|
||||
const { lookupAniList } = require('./metadata/anilist');
|
||||
const { runArchive } = require('./jobs/weebcentral-archive');
|
||||
const { runQueue } = require('./jobs/weebcentral-queue');
|
||||
|
||||
function flag(args, name, fallback) {
|
||||
const index = args.indexOf(name);
|
||||
return index < 0 ? fallback : args[index + 1];
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const command = args.shift();
|
||||
if (command === 'publish') {
|
||||
const file = args[0];
|
||||
if (!file) throw new Error('Usage: publish <manifest.json> [--library path] [--metadata-search title] [--replace]');
|
||||
const manifest = JSON.parse(await fs.readFile(path.resolve(file), 'utf8'));
|
||||
const metadataSearch = flag(args, '--metadata-search');
|
||||
if (metadataSearch) {
|
||||
const metadata = await lookupAniList(metadataSearch);
|
||||
manifest.series.metadata = { ...metadata, title: manifest.series.title };
|
||||
manifest.series.scraper = { ...(manifest.series.scraper || {}), ids: { ...(manifest.series.scraper?.ids || {}), anilist: metadata['x-anilist']?.id } };
|
||||
}
|
||||
const result = await publishManifest(manifest, { libraryRoot: flag(args, '--library'), replace: args.includes('--replace') });
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
if (command === 'discover-weebcentral') {
|
||||
const url = args[0];
|
||||
if (!url) throw new Error('Usage: discover-weebcentral <series-url> --out manifest.json [--limit 3]');
|
||||
const manifest = await discoverWeebCentral(url, { limit: Number(flag(args, '--limit', 3)) });
|
||||
const out = flag(args, '--out');
|
||||
if (out) await fs.writeFile(path.resolve(out), JSON.stringify(manifest, null, 2) + '\n');
|
||||
else console.log(JSON.stringify(manifest, null, 2));
|
||||
return;
|
||||
}
|
||||
if (command === 'metadata-anilist') {
|
||||
const search = args.join(' ');
|
||||
if (!search) throw new Error('Usage: metadata-anilist <title>');
|
||||
console.log(JSON.stringify(await lookupAniList(search), null, 2));
|
||||
return;
|
||||
}
|
||||
if (command === 'archive-weebcentral') {
|
||||
const url = args[0];
|
||||
if (!url) throw new Error('Usage: archive-weebcentral <series-url> [--library path] [--metadata-search title]');
|
||||
await runArchive({
|
||||
seriesUrl: url,
|
||||
libraryRoot: flag(args, '--library', 'library'),
|
||||
metadataSearch: flag(args, '--metadata-search'),
|
||||
minMs: Number(flag(args, '--min-ms', 8000)),
|
||||
maxMs: Number(flag(args, '--max-ms', 15000)),
|
||||
chapterMinMs: Number(flag(args, '--chapter-min-ms', 30000)),
|
||||
chapterMaxMs: Number(flag(args, '--chapter-max-ms', 60000)),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (command === 'status-weebcentral') {
|
||||
const url = args[0];
|
||||
if (!url) throw new Error('Usage: status-weebcentral <series-url> [--library path]');
|
||||
const sourceId = url.split('/series/')[1]?.split('/')[0];
|
||||
if (!sourceId) throw new Error('Could not read WeebCentral series id');
|
||||
const workRoot = path.resolve(flag(args, '--library', 'library'), `.mcscraper-work-${sourceId}`);
|
||||
const state = JSON.parse(await fs.readFile(path.join(workRoot, 'job.json'), 'utf8'));
|
||||
console.log(JSON.stringify({
|
||||
status: state.status || 'unknown',
|
||||
completed: state.completed?.length || 0,
|
||||
total: state.total || null,
|
||||
percent: state.total ? Math.round((state.completed?.length || 0) * 1000 / state.total) / 10 : null,
|
||||
lastChapter: state.lastChapter || null,
|
||||
current: state.current || null,
|
||||
lastProblem: state.lastProblem || null,
|
||||
lastError: state.lastError || null,
|
||||
problemCount: state.problemCount || 0,
|
||||
rate: state.rate || null,
|
||||
updated: state.updated || null,
|
||||
log: path.join(workRoot, 'job.log'),
|
||||
}, null, 2));
|
||||
return;
|
||||
}
|
||||
if (command === 'run-wishlist') {
|
||||
const entries = [
|
||||
['The Gamer', '01J76XY9DYH4MH2HZG2PPF74A6', 'The-Gamer'],
|
||||
['Kanojo Okarishimasu', '01J76XYC7KP5M48JAY8S4ZNDX5', 'Kanojo-Okarishimasu'],
|
||||
['Toradora', '01J76XY7J4Z4G6E42GED6QEQXZ', 'Toradora'],
|
||||
['Solo Leveling', '01J76XYCPSY3C4BNPBRY8JMCBE', 'Solo-Leveling'],
|
||||
['Gantz', '01J76XY7EREQ4B1Q5Z5RJ749V6', 'Gantz'],
|
||||
['One Piece', '01J76XY7E9FNDZ1DBBM6PBJPFK', 'One-Piece'],
|
||||
].map(([title, id, slug]) => ({ id, title, url: `https://weebcentral.com/series/${id}/${slug}` }));
|
||||
await runQueue({
|
||||
entries,
|
||||
waitForFolder: flag(args, '--wait-for', 'Berserk'),
|
||||
libraryRoot: flag(args, '--library', 'library'),
|
||||
minMs: Number(flag(args, '--min-ms', 3000)),
|
||||
maxMs: Number(flag(args, '--max-ms', 7000)),
|
||||
chapterMinMs: Number(flag(args, '--chapter-min-ms', 12000)),
|
||||
chapterMaxMs: Number(flag(args, '--chapter-max-ms', 25000)),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (command === 'status-queue') {
|
||||
const queueRoot = path.resolve(flag(args, '--library', 'library'), '.mcscraper-queue');
|
||||
const state = JSON.parse(await fs.readFile(path.join(queueRoot, 'queue.json'), 'utf8'));
|
||||
console.log(JSON.stringify({ ...state, log: path.join(queueRoot, 'queue.log') }, null, 2));
|
||||
return;
|
||||
}
|
||||
console.log('Commands:\n publish <manifest.json> [--library path] [--metadata-search title] [--replace]\n discover-weebcentral <url> [--limit 3] [--out file]\n archive-weebcentral <url> [--library path] [--metadata-search title]\n status-weebcentral <url> [--library path]\n run-wishlist [--library path] [--wait-for Berserk]\n status-queue [--library path]\n metadata-anilist <title>');
|
||||
}
|
||||
|
||||
main().catch(error => { console.error(`[scraper] ${error.message}`); process.exitCode = 1; });
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
const http = require('http');
|
||||
const path = require('path');
|
||||
const { ScraperService } = require('./service');
|
||||
const { secret } = require('../runtime/config');
|
||||
|
||||
const HOST = process.env.SCRAPER_HOST || '127.0.0.1';
|
||||
const PORT = Number(process.env.SCRAPER_PORT || 8091);
|
||||
const TOKEN = secret('SCRAPER_TOKEN');
|
||||
const LIBRARY = path.resolve(process.env.LIBRARY || 'library');
|
||||
const DATA = path.resolve(process.env.SCRAPER_DATA || path.join(process.env.DATA || 'data', 'scraper'));
|
||||
|
||||
if (!['127.0.0.1', 'localhost', '::1'].includes(HOST) && !TOKEN) {
|
||||
throw new Error('SCRAPER_TOKEN is required when SCRAPER_HOST is not loopback');
|
||||
}
|
||||
|
||||
function json(res, code, value) {
|
||||
res.writeHead(code, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' });
|
||||
res.end(JSON.stringify(value));
|
||||
}
|
||||
|
||||
async function body(req) {
|
||||
let text = '';
|
||||
for await (const chunk of req) { text += chunk; if (text.length > 1000000) throw Object.assign(new Error('Request too large'), { code: 413 }); }
|
||||
try { return text ? JSON.parse(text) : {}; } catch { throw Object.assign(new Error('Invalid JSON'), { code: 400 }); }
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const service = await new ScraperService({ libraryRoot: LIBRARY, dataRoot: DATA }).init();
|
||||
const server = http.createServer(async (req, res) => {
|
||||
try {
|
||||
const url = new URL(req.url, 'http://localhost');
|
||||
if (url.pathname === '/health') return json(res, 200, { ok: true, status: service.state.status });
|
||||
if (TOKEN && req.headers.authorization !== `Bearer ${TOKEN}`) return json(res, 401, { error: 'Unauthorized' });
|
||||
if (req.method === 'GET' && url.pathname === '/api/state') return json(res, 200, await service.publicState());
|
||||
if (req.method !== 'POST') return json(res, 404, { error: 'Not found' });
|
||||
const input = await body(req);
|
||||
if (url.pathname === '/api/queue') return json(res, 200, { item: await service.enqueue(input), state: await service.publicState() });
|
||||
if (url.pathname === '/api/queue/remove') { await service.remove(String(input.id || '')); return json(res, 200, await service.publicState()); }
|
||||
if (url.pathname === '/api/queue/clear-completed') { const result = await service.clearCompleted(String(input.source || '')); return json(res, 200, { ...result, state: await service.publicState() }); }
|
||||
if (url.pathname === '/api/queue/reorder') { await service.reorder(input.ids); return json(res, 200, await service.publicState()); }
|
||||
if (url.pathname === '/api/control') { await service.control(String(input.action || ''), input.id); return json(res, 200, await service.publicState()); }
|
||||
if (url.pathname === '/api/recovery') return json(res, 200, await service.recover(input));
|
||||
if (url.pathname === '/api/settings') { await service.updateSettings(input); return json(res, 200, await service.publicState()); }
|
||||
if (url.pathname === '/api/discovery/run') { await service.enqueueDiscoveryChecks('manual'); return json(res, 200, await service.publicState()); }
|
||||
if (url.pathname === '/api/library/auto-update') { await service.setMangaAutoUpdate(String(input.id || ''), !!input.enabled); return json(res, 200, await service.publicState()); }
|
||||
if (url.pathname === '/api/legacy/publish-ready') return json(res, 200, { result: await service.publishLegacyReady(), state: await service.publicState() });
|
||||
return json(res, 404, { error: 'Not found' });
|
||||
} catch (error) {
|
||||
console.error('[scraper-daemon]', error.message);
|
||||
json(res, error.code || (/not found/i.test(error.message) ? 404 : 400), { error: error.message });
|
||||
}
|
||||
});
|
||||
server.listen(PORT, HOST, () => console.log(`MangaCircus scraper on http://${HOST}:${PORT} (library: ${LIBRARY}, data: ${DATA})`));
|
||||
|
||||
let closing = false;
|
||||
const close = async () => {
|
||||
if (closing) return; closing = true;
|
||||
server.close(); await service.close(); process.exit(0);
|
||||
};
|
||||
process.on('SIGINT', close);
|
||||
process.on('SIGTERM', close);
|
||||
})().catch(error => { console.error(`[scraper-daemon] ${error.message}`); process.exitCode = 1; });
|
||||
@@ -0,0 +1,215 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { SlowHttp } = require('../lib/http');
|
||||
const { sleep } = require('../lib/delay');
|
||||
const { ArchiveControlError } = require('../lib/control');
|
||||
const { cleanPart } = require('../lib/names');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const { selectByScope } = require('../lib/scope');
|
||||
const { scanLibrary, findManga, shouldDownload, sourceKey } = require('../lib/library-inventory');
|
||||
const { lookupAniList, lookupAniListById } = require('../metadata/anilist');
|
||||
const { mangaDexInfo, discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
|
||||
|
||||
const randomBetween = (min, max) => min + Math.floor(Math.random() * (max - min + 1));
|
||||
const exists = async file => { try { await fs.access(file); return true; } catch { return false; } };
|
||||
|
||||
async function writeJsonAtomic(file, value) {
|
||||
const temp = `${file}.tmp`;
|
||||
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n');
|
||||
await fs.rename(temp, file).catch(async error => {
|
||||
if (!['EPERM', 'EEXIST'].includes(error.code)) throw error;
|
||||
await fs.rm(file, { force: true });
|
||||
await fs.rename(temp, file);
|
||||
});
|
||||
}
|
||||
|
||||
async function renameRetry(from, to) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try { await fs.rename(from, to); return; } catch (error) {
|
||||
if (attempt >= 5 || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
||||
await sleep(200 * (attempt + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runMangaDexArchive(options) {
|
||||
const info = mangaDexInfo(options.seriesUrl);
|
||||
if (!info) throw new Error('Could not read MangaDex title id');
|
||||
const checkControl = () => {
|
||||
const reason = options.shouldStop && options.shouldStop();
|
||||
if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason));
|
||||
};
|
||||
const libraryRoot = path.resolve(options.libraryRoot || 'library');
|
||||
const workRoot = path.join(libraryRoot, `.mcscraper-work-${info.id}`);
|
||||
const stateFile = path.join(workRoot, 'job.json');
|
||||
const logFile = path.join(workRoot, 'job.log');
|
||||
await fs.mkdir(workRoot, { recursive: true });
|
||||
|
||||
const resumed = await exists(stateFile);
|
||||
let state = { version: 2, seriesUrl: options.seriesUrl, source: 'mangadex', publicationMode: 'incremental', completed: [], started: new Date().toISOString() };
|
||||
if (resumed) state = JSON.parse(await fs.readFile(stateFile, 'utf8'));
|
||||
const incremental = state.publicationMode === 'incremental';
|
||||
const completed = new Set(state.completed || []);
|
||||
const event = async (level, message, details = {}) => {
|
||||
const entry = { time: new Date().toISOString(), level, message, ...details };
|
||||
await fs.appendFile(logFile, JSON.stringify(entry) + '\n');
|
||||
state.lastEvent = entry;
|
||||
if (level === 'warning' || level === 'error') {
|
||||
state.lastProblem = entry;
|
||||
state.problemCount = Number(state.problemCount || 0) + 1;
|
||||
}
|
||||
state.updated = entry.time;
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
};
|
||||
|
||||
const minMs = Number(options.minMs);
|
||||
const maxMs = Number(options.maxMs);
|
||||
const chapterMinMs = Number(options.chapterMinMs);
|
||||
const chapterMaxMs = Number(options.chapterMaxMs);
|
||||
const http = new SlowHttp({
|
||||
minMs, maxMs, maxRetries: 2,
|
||||
userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)',
|
||||
onEvent: item => event(item.level, item.message, { status: item.status }),
|
||||
});
|
||||
|
||||
checkControl();
|
||||
console.log(`[mangadex] discovering catalog ${info.id}`);
|
||||
const catalog = await discoverMangaDexCatalog(options.seriesUrl, http);
|
||||
if (options.onCatalog) await options.onCatalog(catalog);
|
||||
if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability.message), { failureKind: 'unavailable' });
|
||||
const specialOnly = catalog.chapters.every(ref => ref.special);
|
||||
const folderTitle = String(specialOnly ? catalog.title : options.title || catalog.title).trim() || catalog.title;
|
||||
console.log(`[mangadex] ${folderTitle}: ${catalog.chapters.length} unique English chapters`);
|
||||
const expectedAniList = catalog.links.al ? String(catalog.links.al) : null;
|
||||
const metadata = expectedAniList
|
||||
? await lookupAniListById(expectedAniList)
|
||||
: options.metadataSearch && !specialOnly ? await lookupAniList(options.metadataSearch) : {};
|
||||
const actualAniList = metadata['x-anilist']?.id ? String(metadata['x-anilist'].id) : null;
|
||||
if (expectedAniList && actualAniList && expectedAniList !== actualAniList) {
|
||||
throw new Error(`AniList identity mismatch: MangaDex ${expectedAniList}, lookup ${actualAniList}`);
|
||||
}
|
||||
const series = {
|
||||
title: folderTitle, ...require('../../reading-direction').fallbackDirection(),
|
||||
sources: [{ name: 'MangaDex', url: options.seriesUrl }],
|
||||
metadata: { ...metadata, title: specialOnly ? catalog.title : metadata.title || folderTitle },
|
||||
coverUrl: metadata['x-anilist']?.coverUrl,
|
||||
scraper: { ids: { mangadex: info.id, ...(expectedAniList ? { anilist: expectedAniList } : {}) } },
|
||||
};
|
||||
|
||||
let inventory = await scanLibrary(libraryRoot);
|
||||
let manga = findManga(inventory, {
|
||||
source: 'mangadex', seriesId: info.id, sourceUrl: options.seriesUrl,
|
||||
anilist: specialOnly ? null : expectedAniList || actualAniList, libraryMangaId: options.libraryMangaId,
|
||||
});
|
||||
if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`);
|
||||
const eligible = incremental
|
||||
? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'mangadex', options.existingPolicy))
|
||||
: catalog.chapters;
|
||||
const refs = incremental ? selectByScope(eligible, options.scope) : eligible;
|
||||
state.status = 'running';
|
||||
state.catalogTotal = catalog.chapters.length;
|
||||
state.total = incremental ? refs.length : catalog.chapters.length;
|
||||
state.jobCompleted = 0;
|
||||
state.scope = options.scope || { mode: 'missing' };
|
||||
state.existingPolicy = options.existingPolicy || 'skip-any';
|
||||
state.rate = { minMs, maxMs, chapterMinMs, chapterMaxMs };
|
||||
state.lastError = null;
|
||||
await event('info', `Catalog ready: ${catalog.chapters.length} chapters; ${refs.length} selected`);
|
||||
|
||||
if (incremental && !refs.length) {
|
||||
state.status = 'complete'; state.current = null;
|
||||
await event('info', 'No eligible chapters to download');
|
||||
return { finalPath: manga?.path || null, publishedCount: 0, selectedCount: 0 };
|
||||
}
|
||||
|
||||
let publishedCount = 0;
|
||||
for (let index = 0; index < refs.length; index++) {
|
||||
checkControl();
|
||||
const ref = refs[index];
|
||||
if (!incremental && completed.has(ref.id)) {
|
||||
if (!incremental) console.log(`[mangadex] skip completed ${ref.number} (${index + 1}/${refs.length})`);
|
||||
continue;
|
||||
}
|
||||
if (incremental && !shouldDownload(manga, ref, 'mangadex', options.existingPolicy)) {
|
||||
state.jobCompleted++;
|
||||
await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number });
|
||||
continue;
|
||||
}
|
||||
let chapter, result;
|
||||
try {
|
||||
state.status = 'inspecting';
|
||||
state.current = { number: ref.number, label: ref.special ? ref.label : null, ...(ref.volume ? { volume: ref.volume } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: ref.pages || null };
|
||||
await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number });
|
||||
chapter = await discoverMangaDexChapter(ref, http);
|
||||
state.status = 'downloading';
|
||||
state.current.expectedPages = chapter.pages.length;
|
||||
await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number });
|
||||
console.log(`[mangadex] download ${ref.number}: ${chapter.pages.length} pages`);
|
||||
result = await publishManifest({ version: 1, series, chapters: [chapter] }, {
|
||||
libraryRoot: incremental ? libraryRoot : workRoot,
|
||||
...(incremental && manga ? { mangaDir: manga.path } : {}),
|
||||
httpClient: http,
|
||||
onPage: async progress => {
|
||||
state.current.downloadedPages = progress.completed;
|
||||
state.updated = new Date().toISOString();
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
checkControl();
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ArchiveControlError) {
|
||||
state.status = error.reason;
|
||||
state.current = null;
|
||||
await event('info', `Archive ${error.reason}`, { chapter: ref.number });
|
||||
throw error;
|
||||
}
|
||||
state.status = 'error';
|
||||
state.lastError = { time: new Date().toISOString(), chapter: ref.number, message: error.message };
|
||||
await event('error', error.message, { chapter: ref.number });
|
||||
throw error;
|
||||
}
|
||||
completed.add(ref.id);
|
||||
state.completed = [...completed];
|
||||
state.jobCompleted = Number(state.jobCompleted || 0) + 1;
|
||||
state.lastChapter = ref.number;
|
||||
state.status = 'running';
|
||||
state.current = null;
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
const published = result.results[0].status === 'published';
|
||||
if (published && incremental) {
|
||||
publishedCount++;
|
||||
if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] };
|
||||
manga.chapters.push({ book: '1', number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
|
||||
if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number });
|
||||
}
|
||||
await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
|
||||
console.log(`[mangadex] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
|
||||
if (index < refs.length - 1) {
|
||||
checkControl();
|
||||
await sleep(randomBetween(chapterMinMs, chapterMaxMs));
|
||||
checkControl();
|
||||
}
|
||||
}
|
||||
|
||||
if (incremental) {
|
||||
state.status = 'complete'; state.current = null;
|
||||
await event('info', `Published ${publishedCount} chapter${publishedCount === 1 ? '' : 's'} incrementally`);
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: refs.length };
|
||||
}
|
||||
|
||||
const stagedManga = path.join(workRoot, cleanPart(folderTitle));
|
||||
const finalManga = path.join(libraryRoot, cleanPart(folderTitle));
|
||||
if (await exists(finalManga)) throw new Error(`Final manga folder already exists: ${finalManga}`);
|
||||
await renameRetry(stagedManga, finalManga);
|
||||
state.status = 'complete';
|
||||
state.current = null;
|
||||
await event('info', `Published ${finalManga}`);
|
||||
await fs.rm(workRoot, { recursive: true, force: true });
|
||||
if (options.onPublished) await options.onPublished({ mangaDir: finalManga, completeManga: true });
|
||||
console.log(`[mangadex] PUBLISHED ${finalManga}`);
|
||||
return { finalPath: finalManga, publishedCount: 1, selectedCount: catalog.chapters.length };
|
||||
}
|
||||
|
||||
module.exports = { runMangaDexArchive };
|
||||
@@ -0,0 +1,220 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { SlowHttp } = require('../lib/http');
|
||||
const { sleep } = require('../lib/delay');
|
||||
const { ArchiveControlError } = require('../lib/control');
|
||||
const { cleanPart } = require('../lib/names');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const { selectByScope } = require('../lib/scope');
|
||||
const { scanLibrary, findManga, shouldDownload, sourceKey } = require('../lib/library-inventory');
|
||||
const { lookupAniList } = require('../metadata/anilist');
|
||||
const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral');
|
||||
|
||||
function randomBetween(min, max) {
|
||||
return min + Math.floor(Math.random() * (max - min + 1));
|
||||
}
|
||||
|
||||
async function exists(file) {
|
||||
try { await fs.access(file); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
async function writeJsonAtomic(file, value) {
|
||||
const temp = `${file}.tmp`;
|
||||
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n');
|
||||
await fs.rename(temp, file).catch(async error => {
|
||||
if (!['EPERM', 'EEXIST'].includes(error.code)) throw error;
|
||||
await fs.rm(file, { force: true });
|
||||
await fs.rename(temp, file);
|
||||
});
|
||||
}
|
||||
|
||||
async function renameRetry(from, to) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try { await fs.rename(from, to); return; } catch (error) {
|
||||
if (attempt >= 5 || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
||||
await sleep(200 * (attempt + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runArchive(options) {
|
||||
const checkControl = () => {
|
||||
const reason = options.shouldStop && options.shouldStop();
|
||||
if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason));
|
||||
};
|
||||
const libraryRoot = path.resolve(options.libraryRoot || 'library');
|
||||
const sourceId = options.seriesUrl.split('/series/')[1]?.split('/')[0];
|
||||
if (!sourceId) throw new Error('Could not read WeebCentral series id');
|
||||
const workRoot = path.join(libraryRoot, `.mcscraper-work-${sourceId}`);
|
||||
const stateFile = path.join(workRoot, 'job.json');
|
||||
const logFile = path.join(workRoot, 'job.log');
|
||||
await fs.mkdir(workRoot, { recursive: true });
|
||||
|
||||
const resumed = await exists(stateFile);
|
||||
let state = { version: 2, seriesUrl: options.seriesUrl, publicationMode: 'incremental', completed: [], started: new Date().toISOString() };
|
||||
if (resumed) state = JSON.parse(await fs.readFile(stateFile, 'utf8'));
|
||||
const incremental = state.publicationMode === 'incremental';
|
||||
const completed = new Set(state.completed || []);
|
||||
const event = async (level, message, details = {}) => {
|
||||
const entry = { time: new Date().toISOString(), level, message, ...details };
|
||||
await fs.appendFile(logFile, JSON.stringify(entry) + '\n');
|
||||
state.lastEvent = entry;
|
||||
if (level === 'warning' || level === 'error') {
|
||||
state.lastProblem = entry;
|
||||
state.problemCount = Number(state.problemCount || 0) + 1;
|
||||
}
|
||||
state.updated = entry.time;
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
};
|
||||
const http = new SlowHttp({
|
||||
minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2,
|
||||
onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }),
|
||||
});
|
||||
|
||||
checkControl();
|
||||
console.log(`[archive] discovering catalog ${options.seriesUrl}`);
|
||||
const catalog = await discoverWeebCatalog(options.seriesUrl, http);
|
||||
if (options.onCatalog) await options.onCatalog(catalog);
|
||||
if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || 'WeebCentral lists no chapter links for this series.'), {
|
||||
failureKind: catalog.chapterLinks ? 'adapter' : 'unavailable',
|
||||
});
|
||||
const keys = new Map();
|
||||
for (const chapter of catalog.chapters) {
|
||||
const key = `${chapter.book?.number || 1}:${chapter.number}`;
|
||||
const prior = keys.get(key);
|
||||
if (prior) throw new Error(`Ambiguous mapped chapter ${key}: ${prior.url} and ${chapter.url}`);
|
||||
keys.set(key, chapter);
|
||||
}
|
||||
console.log(`[archive] ${catalog.title}: ${catalog.chapters.length} source chapters`);
|
||||
const metadata = options.metadataSearch ? await lookupAniList(options.metadataSearch) : {};
|
||||
const series = {
|
||||
title: catalog.title, ...require('../../reading-direction').fallbackDirection(),
|
||||
sources: [{ name: 'WeebCentral', url: options.seriesUrl }],
|
||||
metadata: { ...metadata, title: catalog.title },
|
||||
coverUrl: metadata['x-anilist']?.coverUrl,
|
||||
scraper: { ids: { weebcentral: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } },
|
||||
};
|
||||
|
||||
let inventory = await scanLibrary(libraryRoot);
|
||||
let manga = findManga(inventory, {
|
||||
source: 'weebcentral', seriesId: sourceId, sourceUrl: options.seriesUrl,
|
||||
anilist: metadata['x-anilist']?.id, libraryMangaId: options.libraryMangaId,
|
||||
});
|
||||
if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`);
|
||||
const eligible = incremental
|
||||
? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'weebcentral', options.existingPolicy))
|
||||
: catalog.chapters;
|
||||
const refs = incremental ? selectByScope(eligible, options.scope) : eligible;
|
||||
state.status = 'running';
|
||||
state.catalogTotal = catalog.chapters.length;
|
||||
state.total = incremental ? refs.length : catalog.chapters.length;
|
||||
state.jobCompleted = 0;
|
||||
state.scope = options.scope || { mode: 'missing' };
|
||||
state.existingPolicy = options.existingPolicy || 'skip-any';
|
||||
state.rate = { minMs: options.minMs, maxMs: options.maxMs, chapterMinMs: options.chapterMinMs, chapterMaxMs: options.chapterMaxMs };
|
||||
state.lastError = null;
|
||||
await event('info', `Catalog ready: ${catalog.chapters.length} chapters; ${refs.length} selected`);
|
||||
|
||||
if (incremental && !refs.length) {
|
||||
state.status = 'complete'; state.current = null;
|
||||
await event('info', 'No eligible chapters to download');
|
||||
return { finalPath: manga?.path || null, publishedCount: 0, selectedCount: 0 };
|
||||
}
|
||||
|
||||
let publishedCount = 0;
|
||||
for (let index = 0; index < refs.length; index++) {
|
||||
checkControl();
|
||||
const ref = refs[index];
|
||||
if (!incremental && completed.has(ref.id)) {
|
||||
if (!incremental) console.log(`[archive] skip completed ${ref.number} (${index + 1}/${refs.length})`);
|
||||
continue;
|
||||
}
|
||||
if (incremental && !shouldDownload(manga, ref, 'weebcentral', options.existingPolicy)) {
|
||||
state.jobCompleted++;
|
||||
await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number });
|
||||
continue;
|
||||
}
|
||||
console.log(`[archive] inspect ${ref.number} (${index + 1}/${refs.length})`);
|
||||
let chapter;
|
||||
let result;
|
||||
try {
|
||||
state.status = 'inspecting';
|
||||
state.current = { number: ref.number, ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: null };
|
||||
await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number });
|
||||
chapter = await discoverWeebChapter(ref, http);
|
||||
state.status = 'downloading';
|
||||
state.current.expectedPages = chapter.pages.length;
|
||||
await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number });
|
||||
console.log(`[archive] download ${ref.number}: ${chapter.pages.length} pages`);
|
||||
result = await publishManifest({ version: 1, series, chapters: [chapter] }, {
|
||||
libraryRoot: incremental ? libraryRoot : workRoot,
|
||||
...(incremental && manga ? { mangaDir: manga.path } : {}),
|
||||
httpClient: http,
|
||||
onPage: async progress => {
|
||||
state.current.downloadedPages = progress.completed;
|
||||
state.updated = new Date().toISOString();
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
checkControl();
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ArchiveControlError) {
|
||||
state.status = error.reason;
|
||||
state.current = null;
|
||||
await event('info', `Archive ${error.reason}`, { chapter: ref.number });
|
||||
throw error;
|
||||
}
|
||||
state.status = 'error';
|
||||
state.lastError = { time: new Date().toISOString(), chapter: ref.number, message: error.message };
|
||||
await event('error', error.message, { chapter: ref.number });
|
||||
throw error;
|
||||
}
|
||||
completed.add(ref.id);
|
||||
state.completed = [...completed];
|
||||
state.jobCompleted = Number(state.jobCompleted || 0) + 1;
|
||||
state.lastChapter = ref.number;
|
||||
state.updated = new Date().toISOString();
|
||||
state.total = incremental ? refs.length : catalog.chapters.length;
|
||||
state.status = 'running';
|
||||
state.current = null;
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
const published = result.results[0].status === 'published';
|
||||
if (published && incremental) {
|
||||
publishedCount++;
|
||||
if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] };
|
||||
manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
|
||||
if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number });
|
||||
}
|
||||
await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
|
||||
console.log(`[archive] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
|
||||
if (index < refs.length - 1) {
|
||||
checkControl();
|
||||
const pause = randomBetween(options.chapterMinMs, options.chapterMaxMs);
|
||||
console.log(`[archive] chapter pause ${Math.round(pause / 1000)}s`);
|
||||
await sleep(pause);
|
||||
checkControl();
|
||||
}
|
||||
}
|
||||
|
||||
if (incremental) {
|
||||
state.status = 'complete'; state.current = null;
|
||||
await event('info', `Published ${publishedCount} chapter${publishedCount === 1 ? '' : 's'} incrementally`);
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: refs.length };
|
||||
}
|
||||
|
||||
const stagedManga = path.join(workRoot, cleanPart(catalog.title));
|
||||
const finalManga = path.join(libraryRoot, cleanPart(catalog.title));
|
||||
if (await exists(finalManga)) throw new Error(`Final manga folder already exists: ${finalManga}`);
|
||||
await renameRetry(stagedManga, finalManga);
|
||||
state.status = 'complete';
|
||||
state.current = null;
|
||||
await event('info', `Published ${finalManga}`);
|
||||
await fs.rm(workRoot, { recursive: true, force: true });
|
||||
if (options.onPublished) await options.onPublished({ mangaDir: finalManga, completeManga: true });
|
||||
console.log(`[archive] PUBLISHED ${finalManga}`);
|
||||
return { finalPath: finalManga, publishedCount: 1, selectedCount: catalog.chapters.length };
|
||||
}
|
||||
|
||||
module.exports = { runArchive };
|
||||
@@ -0,0 +1,124 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { runArchive } = require('./weebcentral-archive');
|
||||
const { sleep } = require('../lib/delay');
|
||||
|
||||
async function exists(file) {
|
||||
try { await fs.access(file); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
async function writeJsonAtomic(file, value) {
|
||||
const temp = `${file}.tmp`;
|
||||
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n');
|
||||
await fs.rename(temp, file).catch(async error => {
|
||||
if (!['EPERM', 'EEXIST'].includes(error.code)) throw error;
|
||||
await fs.rm(file, { force: true });
|
||||
await fs.rename(temp, file);
|
||||
});
|
||||
}
|
||||
|
||||
async function acquireLock(lockFile) {
|
||||
try {
|
||||
const handle = await fs.open(lockFile, 'wx');
|
||||
await handle.writeFile(String(process.pid));
|
||||
await handle.close();
|
||||
return;
|
||||
} catch (error) {
|
||||
if (error.code !== 'EEXIST') throw error;
|
||||
}
|
||||
const priorPid = Number((await fs.readFile(lockFile, 'utf8').catch(() => '')).trim());
|
||||
try {
|
||||
if (priorPid) process.kill(priorPid, 0);
|
||||
throw new Error(`Queue runner is already active (PID ${priorPid || 'unknown'})`);
|
||||
} catch (error) {
|
||||
if (error.message.startsWith('Queue runner is already active')) throw error;
|
||||
await fs.rm(lockFile, { force: true });
|
||||
const handle = await fs.open(lockFile, 'wx');
|
||||
await handle.writeFile(String(process.pid));
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runQueue(options) {
|
||||
const libraryRoot = path.resolve(options.libraryRoot || 'library');
|
||||
const queueRoot = path.join(libraryRoot, '.mcscraper-queue');
|
||||
const stateFile = path.join(queueRoot, 'queue.json');
|
||||
const logFile = path.join(queueRoot, 'queue.log');
|
||||
const lockFile = path.join(queueRoot, 'queue.lock');
|
||||
await fs.mkdir(queueRoot, { recursive: true });
|
||||
await acquireLock(lockFile);
|
||||
|
||||
let state = {
|
||||
version: 1,
|
||||
status: 'waiting',
|
||||
waitForFolder: options.waitForFolder || null,
|
||||
entries: options.entries,
|
||||
completed: [],
|
||||
created: new Date().toISOString(),
|
||||
};
|
||||
if (await exists(stateFile)) {
|
||||
const saved = JSON.parse(await fs.readFile(stateFile, 'utf8'));
|
||||
state = { ...state, ...saved, entries: saved.entries?.length ? saved.entries : options.entries };
|
||||
}
|
||||
state.pid = process.pid;
|
||||
|
||||
const event = async (level, message, details = {}) => {
|
||||
const entry = { time: new Date().toISOString(), level, message, ...details };
|
||||
await fs.appendFile(logFile, JSON.stringify(entry) + '\n');
|
||||
state.lastEvent = entry;
|
||||
state.updated = entry.time;
|
||||
if (level === 'error') state.lastError = entry;
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
};
|
||||
|
||||
const cleanup = async () => { await fs.rm(lockFile, { force: true }).catch(() => {}); };
|
||||
process.once('exit', () => {});
|
||||
try {
|
||||
await event('info', `Queue runner started (PID ${process.pid})`);
|
||||
if (state.waitForFolder) {
|
||||
const prerequisite = path.join(libraryRoot, state.waitForFolder);
|
||||
state.status = 'waiting';
|
||||
await event('info', `Waiting for ${state.waitForFolder} to finish`);
|
||||
while (!(await exists(prerequisite))) await sleep(60000);
|
||||
await event('info', `${state.waitForFolder} is complete; starting queue`);
|
||||
}
|
||||
|
||||
const completed = new Set(state.completed || []);
|
||||
for (let index = 0; index < state.entries.length; index++) {
|
||||
const item = state.entries[index];
|
||||
if (completed.has(item.id)) continue;
|
||||
state.status = 'downloading';
|
||||
state.current = { ...item, position: index + 1, total: state.entries.length };
|
||||
state.lastError = null;
|
||||
await event('info', `Starting ${item.title}`, { title: item.title });
|
||||
try {
|
||||
await runArchive({
|
||||
seriesUrl: item.url,
|
||||
libraryRoot,
|
||||
metadataSearch: item.metadataSearch || item.title,
|
||||
minMs: options.minMs,
|
||||
maxMs: options.maxMs,
|
||||
chapterMinMs: options.chapterMinMs,
|
||||
chapterMaxMs: options.chapterMaxMs,
|
||||
});
|
||||
} catch (error) {
|
||||
state.status = 'error';
|
||||
await event('error', error.message, { title: item.title });
|
||||
throw error;
|
||||
}
|
||||
completed.add(item.id);
|
||||
state.completed = [...completed];
|
||||
state.current = null;
|
||||
await event('info', `Completed ${item.title}`, { title: item.title });
|
||||
}
|
||||
state.status = 'complete';
|
||||
state.current = null;
|
||||
await event('info', 'Queue complete');
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { runQueue };
|
||||
@@ -0,0 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
class ArchiveControlError extends Error {
|
||||
constructor(reason = 'paused') {
|
||||
super(`Archive ${reason}`);
|
||||
this.name = 'ArchiveControlError';
|
||||
this.reason = reason;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { ArchiveControlError };
|
||||
@@ -0,0 +1,27 @@
|
||||
'use strict';
|
||||
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
class PoliteGate {
|
||||
constructor({ minMs = 8000, maxMs = 15000 } = {}) {
|
||||
if (!Number.isFinite(minMs) || !Number.isFinite(maxMs) || minMs < 0 || maxMs < minMs) {
|
||||
throw new Error('Invalid delay range');
|
||||
}
|
||||
this.minMs = minMs;
|
||||
this.maxMs = maxMs;
|
||||
this.lastRequestAt = 0;
|
||||
}
|
||||
|
||||
async wait() {
|
||||
if (!this.lastRequestAt) return;
|
||||
const target = this.minMs + Math.floor(Math.random() * (this.maxMs - this.minMs + 1));
|
||||
const remaining = target - (Date.now() - this.lastRequestAt);
|
||||
if (remaining > 0) await sleep(remaining);
|
||||
}
|
||||
|
||||
mark() {
|
||||
this.lastRequestAt = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { PoliteGate, sleep };
|
||||
@@ -0,0 +1,23 @@
|
||||
'use strict';
|
||||
|
||||
function safeMessage(error) {
|
||||
return String(error?.message || error || 'Unknown error').replace(/https?:\/\/[^\s]+/gi, raw => {
|
||||
try { const u = new URL(raw); return u.origin + u.pathname; } catch { return '[source URL]'; }
|
||||
}).slice(0, 1500);
|
||||
}
|
||||
function classifyFailure(error) {
|
||||
const message = safeMessage(error);
|
||||
const status = Number(error?.status || /(?:HTTP|returned)\s+(\d{3})/i.exec(message)?.[1]);
|
||||
const code = error?.code || error?.cause?.code;
|
||||
const failure = (kind, scope, retryable = false) => ({ kind, scope, retryable, message, ...(status ? { httpStatus: status } : {}) });
|
||||
if ([401, 403, 429].includes(status) || /captcha|cloudflare|challenge|access.denied|account.ban|paywall|login required/i.test(message)) return failure('access', 'source');
|
||||
if (['ENOSPC', 'EROFS', 'EACCES', 'EPERM', 'EIO'].includes(code)) return failure('storage', 'global');
|
||||
if (error?.failureKind === 'unavailable' || /No downloadable English|title was not found|HTTP 404/i.test(message)) return failure('unavailable', 'job');
|
||||
if (error?.failureKind === 'transient' || status >= 500 || /network|fetch failed|timed? ?out|timeout|socket|connection reset/i.test(message)
|
||||
|| ['ETIMEDOUT', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT'].includes(code)) return failure('transient', 'source', true);
|
||||
return failure('adapter', 'job');
|
||||
}
|
||||
function progressKey(job) {
|
||||
return JSON.stringify([job?.completed || 0, job?.total || 0, job?.current?.sourceId || '', job?.current?.downloadedPages || 0, job?.lastChapter || '']);
|
||||
}
|
||||
module.exports = { classifyFailure, safeMessage, progressKey };
|
||||
@@ -0,0 +1,52 @@
|
||||
'use strict';
|
||||
|
||||
const { PoliteGate, sleep } = require('./delay');
|
||||
|
||||
class StopScrapeError extends Error {}
|
||||
|
||||
class SlowHttp {
|
||||
constructor(options = {}) {
|
||||
this.gate = new PoliteGate(options);
|
||||
this.userAgent = options.userAgent || 'MangaCircus-Scraper/1.0 (personal archival; sequential requests)';
|
||||
this.maxRetries = options.maxRetries ?? 2;
|
||||
this.onEvent = options.onEvent;
|
||||
this.timeoutMs = options.timeoutMs || 60000;
|
||||
}
|
||||
|
||||
async get(url, options = {}) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
await this.gate.wait();
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(url, {
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
headers: { 'user-agent': this.userAgent, accept: options.accept || '*/*', ...(options.headers || {}) },
|
||||
});
|
||||
this.gate.mark();
|
||||
} catch (error) {
|
||||
this.gate.mark();
|
||||
if (attempt >= this.maxRetries) throw new Error(`Network request failed after ${attempt + 1} attempts: ${url}`, { cause: error });
|
||||
const backoff = 30000 * (2 ** attempt);
|
||||
const message = `Network failure; retrying in ${backoff / 1000}s (${attempt + 1}/${this.maxRetries})`;
|
||||
console.warn(`[http] ${message}`);
|
||||
if (this.onEvent) await this.onEvent({ level: 'warning', message, url });
|
||||
await sleep(backoff);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ([401, 403, 429].includes(response.status)) {
|
||||
if (this.onEvent) await this.onEvent({ level: 'error', message: `Host returned ${response.status}; stopping`, url, status: response.status });
|
||||
throw Object.assign(new StopScrapeError(`Host returned ${response.status}; stop and require human review: ${url}`), { status: response.status });
|
||||
}
|
||||
if (response.status >= 500 && attempt < this.maxRetries) {
|
||||
await sleep(30000 * (2 ** attempt));
|
||||
continue;
|
||||
}
|
||||
if (!response.ok) throw Object.assign(new Error(`HTTP ${response.status}: ${url}`), { status: response.status });
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { SlowHttp, StopScrapeError };
|
||||
@@ -0,0 +1,16 @@
|
||||
'use strict';
|
||||
|
||||
const TYPES = [
|
||||
{ ext: 'jpg', test: b => b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
|
||||
{ ext: 'png', test: b => b.length >= 8 && b.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) },
|
||||
{ ext: 'gif', test: b => b.length >= 6 && /^GIF8[79]a$/.test(b.subarray(0, 6).toString('ascii')) },
|
||||
{ ext: 'webp', test: b => b.length >= 12 && b.subarray(0, 4).toString('ascii') === 'RIFF' && b.subarray(8, 12).toString('ascii') === 'WEBP' },
|
||||
{ ext: 'bmp', test: b => b.length >= 2 && b.subarray(0, 2).toString('ascii') === 'BM' },
|
||||
{ ext: 'avif', test: b => b.length >= 12 && b.subarray(4, 12).toString('ascii').includes('ftyp') && /avif|avis/.test(b.subarray(8, 32).toString('ascii')) },
|
||||
];
|
||||
|
||||
function imageExtension(buffer) {
|
||||
return TYPES.find(type => type.test(buffer))?.ext || null;
|
||||
}
|
||||
|
||||
module.exports = { imageExtension };
|
||||
@@ -0,0 +1,30 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
|
||||
async function writeJsonAtomic(file, value, io = fs) {
|
||||
const folder = path.dirname(file), name = path.basename(file);
|
||||
const temp = path.join(folder, `.${name}.${crypto.randomUUID()}.tmp`);
|
||||
await io.writeFile(temp, JSON.stringify(value, null, 2) + '\n');
|
||||
try {
|
||||
try { await io.rename(temp, file); return; }
|
||||
catch (error) {
|
||||
if (!['EPERM', 'EACCES', 'EBUSY', 'EEXIST'].includes(error.code)) throw error;
|
||||
}
|
||||
// Windows may refuse rename-over-existing. Keep the old JSON recoverable
|
||||
// until the replacement is safely in place; never delete it first.
|
||||
const backup = path.join(folder, `.old-${name}-${crypto.randomUUID()}`);
|
||||
await io.rename(file, backup);
|
||||
try { await io.rename(temp, file); }
|
||||
catch (error) {
|
||||
try { await io.rename(backup, file); }
|
||||
catch (restoreError) { error.message += `; previous JSON remains at ${backup}: ${restoreError.message}`; }
|
||||
throw error;
|
||||
}
|
||||
await io.rm(backup, { force: true });
|
||||
} finally { await io.rm(temp, { force: true }); }
|
||||
}
|
||||
|
||||
module.exports = { writeJsonAtomic };
|
||||
@@ -0,0 +1,179 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { imageExtension } = require('./images');
|
||||
const { assertInside } = require('./names');
|
||||
|
||||
const PAGE = /^(\d{4,5})(?:_[a-z0-9]+)*\.(jpe?g|png|webp|gif|avif|bmp)$/i;
|
||||
const BOOK = /^(?:v|s|x)\d{2,3}(?: |$)/i;
|
||||
const CHAPTER = /^(?:c\d{4}(?:\.\d*[1-9])?|x\d{2})(?: |$)/i;
|
||||
const OWN_STAGE = /^\.staging-mcscraper-[a-f0-9]+$/;
|
||||
|
||||
async function exists(target) {
|
||||
try { await fs.access(target); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
async function renameRetry(from, to, tries = 5) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try { await fs.rename(from, to); return; } catch (error) {
|
||||
if (attempt >= tries || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
||||
await new Promise(resolve => setTimeout(resolve, 200 * (attempt + 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function writeJsonAtomic(file, value) {
|
||||
const temp = `${file}.migration.tmp`;
|
||||
const old = `${file}.migration.old`;
|
||||
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o644 });
|
||||
await fs.rm(old, { force: true });
|
||||
await renameRetry(file, old);
|
||||
try {
|
||||
await renameRetry(temp, file);
|
||||
} catch (error) {
|
||||
await renameRetry(old, file).catch(() => {});
|
||||
await fs.rm(temp, { force: true });
|
||||
throw error;
|
||||
}
|
||||
await fs.rm(old, { force: true }).catch(() => {});
|
||||
}
|
||||
|
||||
async function directories(folder) {
|
||||
return (await fs.readdir(folder, { withFileTypes: true })).filter(entry => entry.isDirectory());
|
||||
}
|
||||
|
||||
async function validatePage(file, expected) {
|
||||
const handle = await fs.open(file, 'r');
|
||||
try {
|
||||
const prefix = Buffer.alloc(16);
|
||||
const { bytesRead } = await handle.read(prefix, 0, prefix.length, 0);
|
||||
const actual = imageExtension(prefix.subarray(0, bytesRead));
|
||||
if (!actual) throw new Error(`Unsupported image bytes: ${file}`);
|
||||
const normalized = expected.toLowerCase() === 'jpeg' ? 'jpg' : expected.toLowerCase();
|
||||
if (actual !== normalized) throw new Error(`Image extension does not match bytes: ${file}`);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function validateChapter(folder) {
|
||||
const entries = await fs.readdir(folder, { withFileTypes: true });
|
||||
if (entries.some(entry => entry.isDirectory())) throw new Error(`Chapter contains a subdirectory: ${folder}`);
|
||||
const chapterFile = path.join(folder, 'chapter.json');
|
||||
const chapter = JSON.parse(await fs.readFile(chapterFile, 'utf8'));
|
||||
if (!chapter.source?.name || !chapter.source?.url) throw new Error(`Invalid chapter source: ${chapterFile}`);
|
||||
const pages = entries.filter(entry => entry.isFile() && entry.name !== 'chapter.json').map(entry => {
|
||||
const match = PAGE.exec(entry.name);
|
||||
if (!match) throw new Error(`Invalid chapter file: ${path.join(folder, entry.name)}`);
|
||||
return { name: entry.name, number: Number(match[1]), extension: match[2] };
|
||||
}).sort((a, b) => a.number - b.number);
|
||||
if (!pages.length) throw new Error(`Chapter has no pages: ${folder}`);
|
||||
for (let index = 0; index < pages.length; index++) {
|
||||
if (pages[index].number !== index + 1) throw new Error(`Non-contiguous pages in ${folder}`);
|
||||
await validatePage(path.join(folder, pages[index].name), pages[index].extension);
|
||||
}
|
||||
return pages.length;
|
||||
}
|
||||
|
||||
async function inspectLegacyWork({ libraryRoot, seriesId }) {
|
||||
const id = String(seriesId || '');
|
||||
if (!/^[A-Za-z0-9-]+$/.test(id)) throw new Error(`Unsafe legacy series id: ${id}`);
|
||||
const root = path.resolve(libraryRoot);
|
||||
const workRoot = assertInside(root, path.join(root, `.mcscraper-work-${id}`));
|
||||
const stateFile = path.join(workRoot, 'job.json');
|
||||
if (!(await exists(stateFile))) return null;
|
||||
const state = JSON.parse(await fs.readFile(stateFile, 'utf8'));
|
||||
if (state.publicationMode === 'incremental') return null;
|
||||
const mangaDirs = (await directories(workRoot)).filter(entry => !entry.name.startsWith('.'));
|
||||
if (!mangaDirs.length) return null;
|
||||
if (mangaDirs.length !== 1) throw new Error(`Legacy work ${id} must contain exactly one manga folder`);
|
||||
const mangaDir = path.join(workRoot, mangaDirs[0].name);
|
||||
const target = assertInside(root, path.join(root, mangaDirs[0].name));
|
||||
if (await exists(target)) throw new Error(`Cannot publish legacy work; target already exists: ${target}`);
|
||||
const meta = JSON.parse(await fs.readFile(path.join(mangaDir, 'manga.json'), 'utf8'));
|
||||
if (!meta.title || !Array.isArray(meta.sources) || !meta.sources.length) throw new Error(`Invalid legacy manga metadata: ${mangaDir}`);
|
||||
|
||||
let chapterCount = 0, pageCount = 0;
|
||||
const incompleteStages = [];
|
||||
for (const book of await directories(mangaDir)) {
|
||||
if (book.name.startsWith('.')) throw new Error(`Unexpected hidden folder remains in ${mangaDir}`);
|
||||
if (!BOOK.test(book.name)) throw new Error(`Invalid legacy book folder: ${book.name}`);
|
||||
const bookDir = path.join(mangaDir, book.name);
|
||||
for (const chapter of await directories(bookDir)) {
|
||||
if (chapter.name.startsWith('.')) {
|
||||
if (!OWN_STAGE.test(chapter.name)) throw new Error(`Unexpected hidden folder remains in ${bookDir}: ${chapter.name}`);
|
||||
incompleteStages.push(path.join(bookDir, chapter.name));
|
||||
continue;
|
||||
}
|
||||
if (!CHAPTER.test(chapter.name)) throw new Error(`Invalid legacy chapter folder: ${chapter.name}`);
|
||||
pageCount += await validateChapter(path.join(bookDir, chapter.name));
|
||||
chapterCount++;
|
||||
}
|
||||
}
|
||||
const completedCount = new Set(state.completed || []).size;
|
||||
if (!chapterCount || chapterCount !== completedCount) {
|
||||
throw new Error(`Legacy work ${id} has ${chapterCount} complete folders but ${completedCount} completed checkpoint entries`);
|
||||
}
|
||||
return {
|
||||
id, root, workRoot, stateFile, state, mangaDir, target,
|
||||
mangaId: path.basename(target), chapterCount, pageCount, incompleteStages,
|
||||
};
|
||||
}
|
||||
|
||||
async function promoteLegacyWork(inspection) {
|
||||
const { stateFile, mangaDir, target, state, workRoot } = inspection;
|
||||
const quarantineRoot = path.join(workRoot, '.legacy-incomplete');
|
||||
const quarantined = [];
|
||||
if (inspection.incompleteStages.length) await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o755 });
|
||||
for (const stage of inspection.incompleteStages) {
|
||||
const relative = path.relative(mangaDir, stage).split(path.sep).join('__');
|
||||
const quarantine = path.join(quarantineRoot, relative);
|
||||
await renameRetry(stage, quarantine);
|
||||
quarantined.push({ stage, quarantine });
|
||||
}
|
||||
try {
|
||||
await renameRetry(mangaDir, target);
|
||||
} catch (error) {
|
||||
for (const item of quarantined.reverse()) await renameRetry(item.quarantine, item.stage).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
const migrated = {
|
||||
...state,
|
||||
publicationMode: 'incremental',
|
||||
status: 'paused',
|
||||
current: null,
|
||||
jobCompleted: 0,
|
||||
total: 0,
|
||||
migratedToIncremental: {
|
||||
time: new Date().toISOString(),
|
||||
mangaId: inspection.mangaId,
|
||||
chaptersPublished: inspection.chapterCount,
|
||||
pagesPublished: inspection.pageCount,
|
||||
incompleteStagesQuarantined: quarantined.length,
|
||||
},
|
||||
};
|
||||
try {
|
||||
await writeJsonAtomic(stateFile, migrated);
|
||||
} catch (error) {
|
||||
await renameRetry(target, mangaDir).catch(() => {});
|
||||
for (const item of quarantined.reverse()) await renameRetry(item.quarantine, item.stage).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
await fs.appendFile(path.join(workRoot, 'job.log'), JSON.stringify({
|
||||
time: migrated.migratedToIncremental.time,
|
||||
level: 'info',
|
||||
message: `Published ${inspection.chapterCount} completed legacy chapters and switched to incremental mode`,
|
||||
mangaId: inspection.mangaId,
|
||||
}) + '\n').catch(() => {});
|
||||
return {
|
||||
seriesId: inspection.id,
|
||||
mangaId: inspection.mangaId,
|
||||
finalPath: target,
|
||||
chaptersPublished: inspection.chapterCount,
|
||||
pagesPublished: inspection.pageCount,
|
||||
incompleteStagesQuarantined: quarantined.length,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { inspectLegacyWork, promoteLegacyWork };
|
||||
@@ -0,0 +1,158 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { assertInside } = require('./names');
|
||||
const { chapterNumber } = require('./scope');
|
||||
const { isSpecialKey, parseSpecialFolder, parseKey } = require('../../chapter-identity');
|
||||
const { writeJsonAtomic } = require('./json-file');
|
||||
|
||||
async function readJson(file) {
|
||||
try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return {}; }
|
||||
}
|
||||
|
||||
async function directories(folder) {
|
||||
try { return (await fs.readdir(folder, { withFileTypes: true })).filter(entry => entry.isDirectory() && !entry.name.startsWith('.')); }
|
||||
catch { return []; }
|
||||
}
|
||||
|
||||
function keyFromName(name) { return parseKey(name)?.key || null; }
|
||||
|
||||
function sourceKey(url, name = '') {
|
||||
try {
|
||||
const host = new URL(String(url)).hostname.replace(/^www\./, '').toLowerCase();
|
||||
if (host === 'weebcentral.com') return 'weebcentral';
|
||||
if (host === 'mangadex.org') return 'mangadex';
|
||||
return host;
|
||||
} catch {}
|
||||
const label = String(name).toLowerCase();
|
||||
if (label.startsWith('mangadex')) return 'mangadex';
|
||||
if (label.startsWith('weebcentral')) return 'weebcentral';
|
||||
return label.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
||||
}
|
||||
|
||||
function autoUpdateFor(meta) {
|
||||
const configured = meta?.['x-scraper']?.autoUpdate;
|
||||
return configured === undefined ? String(meta?.status || '').toLowerCase() === 'ongoing' : !!configured;
|
||||
}
|
||||
|
||||
async function scanLibrary(libraryRoot) {
|
||||
const root = path.resolve(libraryRoot);
|
||||
const mangas = [];
|
||||
for (const mangaEntry of await directories(root)) {
|
||||
const mangaPath = path.join(root, mangaEntry.name);
|
||||
const meta = await readJson(path.join(mangaPath, 'manga.json'));
|
||||
const chapters = [];
|
||||
for (const bookEntry of await directories(mangaPath)) {
|
||||
const book = keyFromName(bookEntry.name);
|
||||
if (!book) continue;
|
||||
const bookPath = path.join(mangaPath, bookEntry.name);
|
||||
for (const versionEntry of await directories(bookPath)) {
|
||||
const number = parseSpecialFolder(versionEntry.name)?.key || keyFromName(versionEntry.name);
|
||||
if (!number) continue;
|
||||
const chapterMeta = await readJson(path.join(bookPath, versionEntry.name, 'chapter.json'));
|
||||
const source = typeof chapterMeta.source === 'string' ? { name: chapterMeta.source, url: '' } : (chapterMeta.source || {});
|
||||
chapters.push({
|
||||
book, number, version: versionEntry.name,
|
||||
sourceName: String(source.name || ''), sourceUrl: String(source.url || ''),
|
||||
sourceKey: sourceKey(source.url, source.name),
|
||||
});
|
||||
}
|
||||
}
|
||||
mangas.push({
|
||||
id: mangaEntry.name, path: mangaPath, title: String(meta.title || mangaEntry.name), meta,
|
||||
status: String(meta.status || 'unknown').toLowerCase(), autoUpdate: autoUpdateFor(meta),
|
||||
ids: { ...(meta['x-scraper']?.ids || {}) }, sources: Array.isArray(meta.sources) ? meta.sources : [], chapters,
|
||||
});
|
||||
}
|
||||
return { root, mangas };
|
||||
}
|
||||
|
||||
function findManga(inventory, identity = {}) {
|
||||
const source = String(identity.source || '');
|
||||
const seriesId = identity.seriesId == null ? '' : String(identity.seriesId);
|
||||
const sourceUrl = String(identity.sourceUrl || '');
|
||||
const anilist = identity.anilist == null ? '' : String(identity.anilist);
|
||||
const explicit = identity.libraryMangaId == null ? '' : String(identity.libraryMangaId);
|
||||
if (explicit) return inventory.mangas.find(manga => manga.id === explicit) || null;
|
||||
if (source && seriesId) {
|
||||
const bySourceId = inventory.mangas.find(manga => String(manga.ids[source] ?? '') === seriesId);
|
||||
if (bySourceId) return bySourceId;
|
||||
}
|
||||
if (sourceUrl) {
|
||||
const byUrl = inventory.mangas.find(manga => manga.sources.some(item => item.url === sourceUrl));
|
||||
if (byUrl) return byUrl;
|
||||
}
|
||||
if (anilist) {
|
||||
const matches = inventory.mangas.filter(manga => String(manga.ids.anilist ?? manga.meta?.['x-anilist']?.id ?? '') === anilist);
|
||||
// AniList identifies the work, not every pilot/edition on a source. Never
|
||||
// silently merge a different listing from that same host into its sibling.
|
||||
if (matches.length !== 1) return null;
|
||||
if (source && seriesId && matches[0].ids[source] != null && String(matches[0].ids[source]) !== seriesId) return null;
|
||||
return matches[0];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function chapterBook(ref, manga) {
|
||||
if (manga?.meta?.['x-scraper']?.strategy === 'volumes' && ref.volume != null) return chapterNumber(ref.volume, 'volume');
|
||||
return chapterNumber(ref.book?.number || 1, 'book');
|
||||
}
|
||||
|
||||
function existingVersions(manga, ref) {
|
||||
if (!manga) return [];
|
||||
const book = chapterBook(ref, manga);
|
||||
const number = isSpecialKey(ref.number) ? String(ref.number).toLowerCase() : chapterNumber(ref.number);
|
||||
return manga.chapters.filter(item => item.book === book && item.number === number);
|
||||
}
|
||||
|
||||
function shouldDownload(manga, ref, source, existingPolicy = 'skip-any') {
|
||||
const versions = existingVersions(manga, ref);
|
||||
if (!versions.length) return true;
|
||||
if (existingPolicy === 'add-source') return !versions.some(item => item.sourceKey === source);
|
||||
return false;
|
||||
}
|
||||
|
||||
function isNewerThanLibrary(manga, ref) {
|
||||
if (isSpecialKey(ref.number)) return !existingVersions(manga, ref).length;
|
||||
if (!manga?.chapters?.length) return true;
|
||||
const regular = manga.chapters.filter(item => !String(item.book).startsWith('x') && /^\d+(?:\.\d+)?$/.test(String(item.number)));
|
||||
if (!regular.length) return true;
|
||||
const latest = regular.reduce((best, item) => {
|
||||
if (!best) return item;
|
||||
return Number(item.book) > Number(best.book)
|
||||
|| (Number(item.book) === Number(best.book) && Number(item.number) > Number(best.number)) ? item : best;
|
||||
}, null);
|
||||
const candidate = { book: chapterBook(ref, manga), number: chapterNumber(ref.number) };
|
||||
return Number(candidate.book) > Number(latest.book)
|
||||
|| (Number(candidate.book) === Number(latest.book) && Number(candidate.number) > Number(latest.number));
|
||||
}
|
||||
|
||||
function summarizeInventory(inventory, supported) {
|
||||
return inventory.mangas.map(manga => ({
|
||||
id: manga.id, title: manga.title, status: manga.status, autoUpdate: manga.autoUpdate,
|
||||
chapters: new Set(manga.chapters.map(item => `${item.book}/${item.number}`)).size,
|
||||
versions: manga.chapters.length,
|
||||
sources: manga.sources.map(source => {
|
||||
const found = supported(source.url);
|
||||
return { name: source.name, url: source.url, adapter: found?.adapter.id || null };
|
||||
}),
|
||||
}));
|
||||
}
|
||||
|
||||
async function setAutoUpdate(libraryRoot, mangaId, enabled) {
|
||||
const root = path.resolve(libraryRoot);
|
||||
const folder = assertInside(root, path.join(root, String(mangaId)));
|
||||
const stat = await fs.stat(folder).catch(() => null);
|
||||
if (!stat?.isDirectory() || path.basename(folder).startsWith('.')) throw new Error('Library manga not found');
|
||||
const file = path.join(folder, 'manga.json');
|
||||
const meta = await readJson(file);
|
||||
meta['x-scraper'] = { ...(meta['x-scraper'] || {}), autoUpdate: !!enabled };
|
||||
await writeJsonAtomic(file, meta);
|
||||
return { id: mangaId, autoUpdate: !!enabled };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
scanLibrary, findManga, existingVersions, shouldDownload, summarizeInventory,
|
||||
isNewerThanLibrary, setAutoUpdate, sourceKey, autoUpdateFor, keyFromName,
|
||||
};
|
||||
@@ -0,0 +1,34 @@
|
||||
'use strict';
|
||||
|
||||
const IMPORTED_FIELDS = ['title', 'altTitles', 'authors', 'status', 'year', 'tags', 'description'];
|
||||
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
||||
|
||||
// The effective fields remain at the top of manga.json. A snapshot of the last
|
||||
// imported values lets later imports refresh unchanged fields without replacing
|
||||
// values somebody edited in the library. Legacy files have no snapshot, so their
|
||||
// existing values are conservatively treated as intentional.
|
||||
function mergeImportedMetadata(prior, imported, fallbackTitle) {
|
||||
const previous = prior['x-metadata']?.imported || {};
|
||||
const result = { ...prior };
|
||||
for (const field of IMPORTED_FIELDS) {
|
||||
if (!Object.hasOwn(imported, field)) continue;
|
||||
if (result[field] === undefined || result[field] === null || result[field] === ''
|
||||
|| (Object.hasOwn(previous, field) && same(result[field], previous[field]))) {
|
||||
result[field] = imported[field];
|
||||
}
|
||||
}
|
||||
if (!result.title) result.title = fallbackTitle;
|
||||
|
||||
const oldId = prior['x-anilist']?.id;
|
||||
const newId = imported['x-anilist']?.id;
|
||||
if (oldId != null && newId != null && String(oldId) !== String(newId)) {
|
||||
throw new Error(`AniList identity conflict: existing ${oldId}, imported ${newId}; review the manga match`);
|
||||
}
|
||||
if (imported['x-anilist']) result['x-anilist'] = { ...(prior['x-anilist'] || {}), ...imported['x-anilist'] };
|
||||
const snapshot = { ...previous };
|
||||
for (const field of IMPORTED_FIELDS) if (Object.hasOwn(imported, field)) snapshot[field] = imported[field];
|
||||
if (Object.keys(snapshot).length) result['x-metadata'] = { ...(prior['x-metadata'] || {}), imported: snapshot };
|
||||
return result;
|
||||
}
|
||||
|
||||
module.exports = { mergeImportedMetadata, IMPORTED_FIELDS };
|
||||
@@ -0,0 +1,50 @@
|
||||
'use strict';
|
||||
|
||||
const path = require('path');
|
||||
const { isSpecialKey } = require('../../chapter-identity');
|
||||
|
||||
const RESERVED = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;
|
||||
|
||||
function cleanPart(value, max = 120) {
|
||||
let result = String(value || '')
|
||||
.normalize('NFC')
|
||||
.replace(/[<>:"/\\|?*\x00-\x1f]/g, '')
|
||||
.replace(/[\[\]]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.replace(/[ .]+$/g, '');
|
||||
if (!result) result = 'Untitled';
|
||||
if (RESERVED.test(result)) result += '-manga';
|
||||
return result.slice(0, max).replace(/[ .]+$/g, '');
|
||||
}
|
||||
|
||||
function chapterKey(value) {
|
||||
const raw = String(value).trim();
|
||||
if (isSpecialKey(raw)) return raw.toLowerCase();
|
||||
if (/^\d+(?:\.\d+)?$/.test(raw)) {
|
||||
const [integer, decimal = ''] = raw.split('.');
|
||||
const normalized = decimal.replace(/0+$/g, '');
|
||||
return `c${integer.padStart(4, '0')}${normalized ? `.${normalized}` : ''}`;
|
||||
}
|
||||
const extra = raw.match(/^x(\d+)$/i);
|
||||
if (extra) return `x${extra[1].padStart(2, '0')}`;
|
||||
throw new Error(`Unsupported chapter number: ${value}`);
|
||||
}
|
||||
|
||||
function chapterFolder(chapter) {
|
||||
const parts = [chapterKey(chapter.number)];
|
||||
if (chapter.title) parts.push(cleanPart(chapter.title));
|
||||
parts.push(`[${cleanPart(chapter.source.name, 50)}]`);
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
function assertInside(parent, child) {
|
||||
const root = path.resolve(parent);
|
||||
const target = path.resolve(child);
|
||||
if (target !== root && !target.startsWith(root + path.sep)) {
|
||||
throw new Error(`Path escapes library root: ${target}`);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
module.exports = { cleanPart, chapterKey, chapterFolder, assertInside };
|
||||
@@ -0,0 +1,279 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const fsp = fs.promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { SlowHttp } = require('./http');
|
||||
const { imageExtension } = require('./images');
|
||||
const { cleanPart, chapterFolder, assertInside } = require('./names');
|
||||
const { sizeFromBuffer, imageSize } = require('../../library');
|
||||
const { directionFields } = require('../../reading-direction');
|
||||
const { mergeImportedMetadata } = require('./metadata-merge');
|
||||
|
||||
const token = () => crypto.randomBytes(5).toString('hex');
|
||||
// Staging is a recovery asset, not disposable garbage. Legacy stages without an
|
||||
// identity must be reviewed manually; never infer a chapter from page filenames.
|
||||
const ownTemp = name => /^\.old-mcscraper-[a-f0-9]+$/.test(name);
|
||||
const digest = value => crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||
const coverFile = name => /^cover\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name);
|
||||
|
||||
async function exists(file) {
|
||||
try { await fsp.access(file); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
async function readMetadata(file) {
|
||||
try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^\uFEFF/, '')); }
|
||||
catch (error) { if (error.code === 'ENOENT') return {}; throw error; }
|
||||
}
|
||||
|
||||
function pageDimensions(size, tags = []) {
|
||||
const rotation = tags.find(tag => /^rot(90|180|270)$/.test(tag));
|
||||
return { w: size?.w || 0, h: size?.h || 0, r: rotation ? Number(rotation.slice(3)) : 0 };
|
||||
}
|
||||
|
||||
async function publishedDimensions(folder) {
|
||||
const pages = [];
|
||||
for (const name of (await fsp.readdir(folder)).sort()) {
|
||||
if (!/\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name)) continue;
|
||||
const tags = name.slice(0, name.lastIndexOf('.')).split('_').slice(1);
|
||||
pages.push(pageDimensions(await imageSize(path.join(folder, name)).catch(() => null), tags));
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
async function renameRetry(from, to, tries = 5) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try { await fsp.rename(from, to); return; } catch (error) {
|
||||
if (attempt >= tries || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
||||
await new Promise(resolve => setTimeout(resolve, 200 * (attempt + 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function writeJsonAtomic(file, value) {
|
||||
const temp = path.join(path.dirname(file), `.${path.basename(file)}.tmp`);
|
||||
await fsp.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o644 });
|
||||
if (!(await exists(file))) return renameRetry(temp, file);
|
||||
try {
|
||||
await renameRetry(temp, file);
|
||||
} catch (error) {
|
||||
if (!['EPERM', 'EBUSY', 'EACCES', 'EEXIST'].includes(error.code)) throw error;
|
||||
const old = path.join(path.dirname(file), `.old-mcscraper-${token()}`);
|
||||
await renameRetry(file, old);
|
||||
try { await renameRetry(temp, file); } catch (moveError) { await renameRetry(old, file); throw moveError; }
|
||||
await fsp.rm(old, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupOwnTemps(folder, maxAgeMs = 6 * 60 * 60 * 1000) {
|
||||
if (!(await exists(folder))) return;
|
||||
const now = Date.now();
|
||||
for (const entry of await fsp.readdir(folder, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || !ownTemp(entry.name)) continue;
|
||||
const target = path.join(folder, entry.name);
|
||||
const stat = await fsp.stat(target);
|
||||
if (now - stat.mtimeMs > maxAgeMs) await fsp.rm(target, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function findManga(libraryRoot, sourceUrls) {
|
||||
if (!(await exists(libraryRoot))) return null;
|
||||
for (const entry of await fsp.readdir(libraryRoot, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name.startsWith('.')) continue;
|
||||
const file = path.join(libraryRoot, entry.name, 'manga.json');
|
||||
try {
|
||||
const json = JSON.parse(await fsp.readFile(file, 'utf8'));
|
||||
if ((json.sources || []).some(source => sourceUrls.has(source.url))) return path.join(libraryRoot, entry.name);
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function validateManifest(manifest) {
|
||||
if (!manifest || manifest.version !== 1) throw new Error('Manifest version must be 1');
|
||||
if (!manifest.series?.title) throw new Error('series.title is required');
|
||||
if (!['rtl', 'ltr', 'ttb'].includes(manifest.series.direction)) throw new Error('series.direction must be rtl, ltr, or ttb');
|
||||
if (!Array.isArray(manifest.series.sources) || !manifest.series.sources.length) throw new Error('series.sources is required');
|
||||
if (!Array.isArray(manifest.chapters) || !manifest.chapters.length) throw new Error('At least one chapter is required');
|
||||
for (const chapter of manifest.chapters) {
|
||||
if (chapter.number == null || chapter.number === '' || !chapter.source?.name || !chapter.source?.url) throw new Error('Every chapter needs number and source {name,url}');
|
||||
if (!Array.isArray(chapter.pages) || !chapter.pages.length) throw new Error(`Chapter ${chapter.number} has no pages`);
|
||||
for (const page of chapter.pages) {
|
||||
if (!!page.url === !!page.path) throw new Error('Every page must have exactly one of url or path');
|
||||
for (const tag of page.tags || []) if (!/^[a-z0-9]+$/.test(tag)) throw new Error(`Invalid page tag: ${tag}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function pageBytes(page, http) {
|
||||
if (page.path) return fsp.readFile(path.resolve(page.path));
|
||||
const response = await http.get(page.url, { headers: page.headers || {} });
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
function validateCoverUrl(value) {
|
||||
const url = new URL(value);
|
||||
if (url.protocol !== 'https:' || !(url.hostname === 'anilist.co' || url.hostname.endsWith('.anilist.co'))) {
|
||||
throw new Error('Series cover must use the AniList HTTPS image host');
|
||||
}
|
||||
return url.href;
|
||||
}
|
||||
|
||||
async function installSeriesCover(mangaDir, coverUrl, http) {
|
||||
const files = await fsp.readdir(mangaDir);
|
||||
if (files.some(coverFile)) return { status: 'existing' };
|
||||
const response = await http.get(validateCoverUrl(coverUrl));
|
||||
const data = Buffer.from(await response.arrayBuffer());
|
||||
if (data.length > 10 * 1024 * 1024) throw new Error('Series cover exceeds 10 MiB');
|
||||
const ext = imageExtension(data);
|
||||
if (!ext) throw new Error('Series cover is not a supported image');
|
||||
const temp = path.join(mangaDir, `.cover-mcscraper-${token()}.${ext}`);
|
||||
const target = path.join(mangaDir, `cover.${ext}`);
|
||||
await fsp.writeFile(temp, data, { mode: 0o644 });
|
||||
try { await renameRetry(temp, target); } catch (error) { await fsp.rm(temp, { force: true }); throw error; }
|
||||
return { status: 'installed', file: path.basename(target) };
|
||||
}
|
||||
|
||||
async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) {
|
||||
// Full manifest fingerprint prevents reusing a page from a changed release.
|
||||
// Store only the hash, never image URLs/headers (which may contain signatures).
|
||||
const identity = digest([chapterFolder(chapter), chapter.source, chapter.pages.map(p => [p.url || path.resolve(p.path), p.tags || []])]);
|
||||
const stage = path.join(bookDir, `.staging-mcscraper-resume-${identity}`);
|
||||
await fsp.mkdir(stage, { recursive: true, mode: 0o755 });
|
||||
const dimensions = [];
|
||||
{
|
||||
for (let index = 0; index < chapter.pages.length; index++) {
|
||||
const tags = (chapter.pages[index].tags || []).map(tag => `_${tag}`).join('');
|
||||
const prefix = `${String(index + 1).padStart(4, '0')}${tags}`;
|
||||
const prior = (await fsp.readdir(stage)).filter(name => name.startsWith(prefix + '.') && !name.endsWith('.tmp'));
|
||||
if (prior.length === 1) {
|
||||
const bytes = await fsp.readFile(path.join(stage, prior[0]));
|
||||
if (imageExtension(bytes) === path.extname(prior[0]).slice(1)) {
|
||||
dimensions.push(pageDimensions(sizeFromBuffer(bytes), chapter.pages[index].tags));
|
||||
if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (prior.length) throw new Error(`Partial page ${index + 1} requires review: invalid or ambiguous image`);
|
||||
const data = await pageBytes(chapter.pages[index], http);
|
||||
const ext = imageExtension(data);
|
||||
if (!ext) throw new Error(`Page ${index + 1} is not a supported image`);
|
||||
dimensions.push(pageDimensions(sizeFromBuffer(data), chapter.pages[index].tags));
|
||||
const target = path.join(stage, `${prefix}.${ext}`);
|
||||
const pending = path.join(stage, `.${prefix}.tmp`);
|
||||
await fsp.writeFile(pending, data, { mode: 0o644 });
|
||||
await renameRetry(pending, target);
|
||||
if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length });
|
||||
}
|
||||
await fsp.writeFile(path.join(stage, 'chapter.json'), JSON.stringify({
|
||||
...priorMeta,
|
||||
title: chapter.title || '', source: chapter.source,
|
||||
added: chapter.added || new Date().toISOString(),
|
||||
priority: Number(chapter.priority || 0), extra: !!chapter.extra,
|
||||
...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId, sourceNumber: chapter.sourceNumber ?? null } : {}),
|
||||
...directionFields([priorMeta, chapter], dimensions),
|
||||
}, null, 2) + '\n', { mode: 0o644 });
|
||||
// A process interruption during a write can leave a hidden pending file.
|
||||
for (const name of await fsp.readdir(stage)) {
|
||||
if (/^\.\d{4,5}(?:_[a-z0-9]+)*\.tmp$/.test(name)) await fsp.rm(path.join(stage, name), { force: true });
|
||||
}
|
||||
return { stage, dimensions };
|
||||
}
|
||||
}
|
||||
|
||||
async function publishStaged(stage, target, replace) {
|
||||
if (!(await exists(target))) return renameRetry(stage, target);
|
||||
if (!replace) { await fsp.rm(stage, { recursive: true, force: true }); return false; }
|
||||
const old = path.join(path.dirname(target), `.old-mcscraper-${token()}`);
|
||||
await renameRetry(target, old);
|
||||
try { await renameRetry(stage, target); } catch (error) { await renameRetry(old, target); throw error; }
|
||||
await fsp.rm(old, { recursive: true, force: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
async function publishManifest(manifest, options = {}) {
|
||||
validateManifest(manifest);
|
||||
const libraryRoot = path.resolve(options.libraryRoot || manifest.libraryRoot || 'library');
|
||||
await fsp.mkdir(libraryRoot, { recursive: true, mode: 0o755 });
|
||||
await cleanupOwnTemps(libraryRoot);
|
||||
const sourceUrls = new Set(manifest.series.sources.map(source => source.url));
|
||||
let mangaDir = options.mangaDir ? assertInside(libraryRoot, options.mangaDir) : await findManga(libraryRoot, sourceUrls);
|
||||
if (mangaDir && !(await exists(mangaDir))) throw new Error(`Target manga folder does not exist: ${mangaDir}`);
|
||||
if (mangaDir && path.basename(mangaDir).startsWith('.')) throw new Error('Target manga folder must be published');
|
||||
const isNew = !mangaDir;
|
||||
const finalName = cleanPart(manifest.series.folder || manifest.series.title);
|
||||
const finalDir = assertInside(libraryRoot, path.join(libraryRoot, finalName));
|
||||
if (!mangaDir && await exists(finalDir)) throw new Error(`Folder already belongs to another work: ${finalDir}`);
|
||||
mangaDir = mangaDir || path.join(libraryRoot, `.staging-mcscraper-resume-${digest([finalName, [...sourceUrls].sort()])}`);
|
||||
await fsp.mkdir(mangaDir, { recursive: true, mode: 0o755 });
|
||||
const http = options.httpClient || new SlowHttp(options.http || manifest.http || {});
|
||||
const results = [];
|
||||
const dimensions = [];
|
||||
let cover = null;
|
||||
const prior = await readMetadata(path.join(mangaDir, 'manga.json'));
|
||||
const owned = manifest.series.metadata || {};
|
||||
// Validate identity before fetching or exposing any chapter pages.
|
||||
const merged = mergeImportedMetadata(prior, owned, manifest.series.title);
|
||||
|
||||
try {
|
||||
for (const chapter of manifest.chapters) {
|
||||
const bookNumber = Number(chapter.book?.number || 1);
|
||||
if (!Number.isInteger(bookNumber) || bookNumber < 1) throw new Error(`Invalid book number for chapter ${chapter.number}`);
|
||||
const bookName = `v${String(bookNumber).padStart(2, '0')}${chapter.book?.title ? ` ${cleanPart(chapter.book.title)}` : ''}`;
|
||||
const bookDir = path.join(mangaDir, bookName);
|
||||
await fsp.mkdir(bookDir, { recursive: true, mode: 0o755 });
|
||||
await cleanupOwnTemps(bookDir);
|
||||
if (chapter.book?.title) {
|
||||
const file = path.join(bookDir, 'book.json'), priorBook = await readMetadata(file);
|
||||
await writeJsonAtomic(file, { ...priorBook, title: chapter.book.title,
|
||||
...(chapter.book.direction || chapter.book.directionOverride
|
||||
? directionFields([priorBook, chapter.book], []) : {}) });
|
||||
}
|
||||
const folder = chapterFolder(chapter);
|
||||
const target = path.join(bookDir, folder);
|
||||
if (await exists(target) && !options.replace) {
|
||||
dimensions.push(...await publishedDimensions(target));
|
||||
results.push({ chapter: chapter.number, status: 'skipped' }); continue;
|
||||
}
|
||||
const priorChapter = await readMetadata(path.join(target, 'chapter.json'));
|
||||
const staged = await stageChapter(bookDir, chapter, http, options.onPage, priorChapter);
|
||||
dimensions.push(...staged.dimensions);
|
||||
await publishStaged(staged.stage, target, !!options.replace);
|
||||
results.push({ chapter: chapter.number, status: 'published', folder });
|
||||
}
|
||||
|
||||
const sources = [...(prior.sources || [])];
|
||||
for (const source of manifest.series.sources) {
|
||||
const index = sources.findIndex(item => item.url === source.url);
|
||||
if (index < 0) sources.push(source); else sources[index] = { ...sources[index], ...source };
|
||||
}
|
||||
const priorScraper = prior['x-scraper'] || {};
|
||||
const nextScraper = { ...priorScraper, strategy: priorScraper.strategy || 'flat', ...(manifest.series.scraper || {}) };
|
||||
await writeJsonAtomic(path.join(mangaDir, 'manga.json'), {
|
||||
...merged,
|
||||
...directionFields([prior, merged, manifest.series], dimensions),
|
||||
sources,
|
||||
'x-scraper': nextScraper,
|
||||
});
|
||||
|
||||
if (manifest.series.coverUrl && !manifest.series.coverAttempted) {
|
||||
manifest.series.coverAttempted = true;
|
||||
try {
|
||||
cover = await installSeriesCover(mangaDir, manifest.series.coverUrl, http);
|
||||
} catch (error) {
|
||||
cover = { status: 'error', message: error.message };
|
||||
console.warn(`[publisher] cover unavailable for ${manifest.series.title}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (isNew) await renameRetry(mangaDir, finalDir);
|
||||
return { mangaDir: isNew ? finalDir : mangaDir, results, cover };
|
||||
} catch (error) {
|
||||
// Preserve hidden validated pages on pause, transient failure or shutdown.
|
||||
// Publication still occurs only after the whole chapter validates.
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { publishManifest, validateManifest, cleanupOwnTemps, installSeriesCover, validateCoverUrl, writeJsonAtomic };
|
||||
@@ -0,0 +1,74 @@
|
||||
'use strict';
|
||||
|
||||
const PUBLIC_MODES = new Set(['missing', 'latest', 'last', 'first', 'chapter', 'after', 'volume']);
|
||||
|
||||
function chapterNumber(value, field = 'chapter') {
|
||||
const raw = String(value ?? '').trim();
|
||||
if (!/^\d+(?:\.\d+)?$/.test(raw)) throw new Error(`${field} must be a numeric chapter`);
|
||||
const [whole, decimal = ''] = raw.split('.');
|
||||
const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
|
||||
const normalizedDecimal = decimal.replace(/0+$/, '');
|
||||
return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
|
||||
}
|
||||
|
||||
function positiveCount(value) {
|
||||
const count = Number(value);
|
||||
if (!Number.isInteger(count) || count < 1 || count > 10000) throw new Error('count must be an integer from 1 to 10000');
|
||||
return count;
|
||||
}
|
||||
|
||||
function normalizeScope(input = {}) {
|
||||
const raw = input && typeof input === 'object' ? input : {};
|
||||
const mode = String(raw.mode || 'missing').toLowerCase();
|
||||
if (mode === 'ids') {
|
||||
if (!Array.isArray(raw.ids) || !raw.ids.length || raw.ids.length > 10000) throw new Error('ids scope requires 1 to 10000 source chapter ids');
|
||||
return { mode, ids: [...new Set(raw.ids.map(value => String(value).trim()).filter(Boolean))] };
|
||||
}
|
||||
if (!PUBLIC_MODES.has(mode)) throw new Error(`Unsupported queue scope: ${mode}`);
|
||||
if (mode === 'last' || mode === 'first') return { mode, count: positiveCount(raw.count) };
|
||||
if (mode === 'chapter' || mode === 'after') return { mode, chapter: chapterNumber(raw.chapter) };
|
||||
if (mode === 'volume') {
|
||||
const volume = String(raw.volume ?? '').trim();
|
||||
if (!/^\d+(?:\.\d+)?$/.test(volume)) throw new Error('volume must be numeric');
|
||||
return { mode, volume: chapterNumber(volume, 'volume') };
|
||||
}
|
||||
return { mode };
|
||||
}
|
||||
|
||||
function volumeOf(ref) {
|
||||
const value = ref.volume ?? ref.book?.number;
|
||||
if (value === undefined || value === null || value === '') return null;
|
||||
try { return chapterNumber(value, 'volume'); } catch { return null; }
|
||||
}
|
||||
|
||||
function selectByScope(chapters, input) {
|
||||
const scope = normalizeScope(input);
|
||||
const list = [...chapters];
|
||||
if (scope.mode === 'missing') return list;
|
||||
if (scope.mode === 'latest') return list.length ? [list[list.length - 1]] : [];
|
||||
if (scope.mode === 'last') return list.slice(-scope.count);
|
||||
if (scope.mode === 'first') return list.slice(0, scope.count);
|
||||
const numbered = list.filter(ref => /^\d+(?:\.\d+)?$/.test(String(ref.number)));
|
||||
if (scope.mode === 'chapter') return numbered.filter(ref => chapterNumber(ref.number) === scope.chapter);
|
||||
if (scope.mode === 'after') return numbered.filter(ref => Number(chapterNumber(ref.number)) > Number(scope.chapter));
|
||||
if (scope.mode === 'volume') return list.filter(ref => volumeOf(ref) === scope.volume);
|
||||
if (scope.mode === 'ids') {
|
||||
const ids = new Set(scope.ids);
|
||||
return list.filter(ref => ids.has(String(ref.id)));
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function scopeLabel(input) {
|
||||
const scope = normalizeScope(input);
|
||||
if (scope.mode === 'missing') return 'all missing chapters';
|
||||
if (scope.mode === 'latest') return 'newest chapter';
|
||||
if (scope.mode === 'last') return `last ${scope.count} chapters`;
|
||||
if (scope.mode === 'first') return `first ${scope.count} chapters`;
|
||||
if (scope.mode === 'chapter') return `chapter ${scope.chapter}`;
|
||||
if (scope.mode === 'after') return `chapters after ${scope.chapter}`;
|
||||
if (scope.mode === 'volume') return `volume ${scope.volume}`;
|
||||
return `${scope.ids.length} discovered chapters`;
|
||||
}
|
||||
|
||||
module.exports = { normalizeScope, selectByScope, scopeLabel, chapterNumber, volumeOf };
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"version": 1,
|
||||
"libraryRoot": "library",
|
||||
"series": {
|
||||
"title": "Example Manga",
|
||||
"direction": "rtl",
|
||||
"sources": [{ "name": "Example", "url": "https://example.invalid/series/1" }],
|
||||
"metadata": { "authors": ["Example Author"], "status": "ongoing" },
|
||||
"scraper": { "ids": { "example": "1" } }
|
||||
},
|
||||
"http": { "minMs": 8000, "maxMs": 15000, "maxRetries": 2 },
|
||||
"chapters": [{
|
||||
"number": "1",
|
||||
"title": "Opening",
|
||||
"added": "2026-01-01T00:00:00Z",
|
||||
"source": { "name": "Example", "url": "https://example.invalid/chapter/1" },
|
||||
"pages": [
|
||||
{ "path": "C:/absolute/path/0001.jpg", "tags": ["cover"] },
|
||||
{ "url": "https://example.invalid/images/2.jpg", "headers": { "referer": "https://example.invalid/chapter/1" } }
|
||||
]
|
||||
}]
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { SlowHttp } = require('../lib/http');
|
||||
|
||||
const QUERY = `query ($search: String) {
|
||||
Media(search: $search, type: MANGA, format: MANGA) {
|
||||
id idMal format title { romaji english native } synonyms
|
||||
staff(perPage: 20) { edges { role node { name { full } } } }
|
||||
status startDate { year } genres description(asHtml: false)
|
||||
countryOfOrigin siteUrl coverImage { extraLarge }
|
||||
relations { edges { relationType node { id type format title { romaji english native } siteUrl } } }
|
||||
}
|
||||
}`;
|
||||
|
||||
const QUERY_BY_ID = `query ($id: Int) {
|
||||
Media(id: $id, type: MANGA, format: MANGA) {
|
||||
id idMal format title { romaji english native } synonyms
|
||||
staff(perPage: 20) { edges { role node { name { full } } } }
|
||||
status startDate { year } genres description(asHtml: false)
|
||||
countryOfOrigin siteUrl coverImage { extraLarge }
|
||||
relations { edges { relationType node { id type format title { romaji english native } siteUrl } } }
|
||||
}
|
||||
}`;
|
||||
|
||||
function stripHtml(value) {
|
||||
return String(value || '').replace(/<br\s*\/?>/gi, '\n').replace(/<[^>]+>/g, '').trim();
|
||||
}
|
||||
|
||||
function metadataFromMedia(media) {
|
||||
return {
|
||||
title: media.title.english || media.title.romaji,
|
||||
altTitles: [...new Set([media.title.romaji, media.title.native, ...(media.synonyms || [])].filter(Boolean))],
|
||||
authors: [...new Set((media.staff?.edges || [])
|
||||
.filter(edge => /^(?:original story|story(?: & art)?|art|illustration)$/i.test(edge.role || ''))
|
||||
.map(edge => edge.node?.name?.full).filter(Boolean))],
|
||||
status: media.status === 'FINISHED' ? 'completed' : media.status === 'RELEASING' ? 'ongoing' : 'unknown',
|
||||
year: media.startDate?.year || undefined,
|
||||
tags: media.genres || [], description: stripHtml(media.description),
|
||||
'x-anilist': { id: media.id, idMal: media.idMal, format: media.format, url: media.siteUrl, coverUrl: media.coverImage?.extraLarge,
|
||||
relations: (media.relations?.edges || []).filter(edge => edge.node?.type === 'MANGA').map(edge => ({ id: edge.node.id, type: edge.relationType,
|
||||
format: edge.node.format, title: edge.node.title?.english || edge.node.title?.romaji || edge.node.title?.native, url: edge.node.siteUrl })) },
|
||||
};
|
||||
}
|
||||
|
||||
async function lookup(query, variables, cacheKey, options = {}) {
|
||||
const cacheDir = path.resolve(options.cacheDir || process.env.METADATA_CACHE || path.join(__dirname, '..', '.cache', 'anilist'));
|
||||
await fs.mkdir(cacheDir, { recursive: true });
|
||||
const cacheFile = path.join(cacheDir, `${cacheKey}.json`);
|
||||
try {
|
||||
const stat = await fs.stat(cacheFile);
|
||||
const cached = JSON.parse(await fs.readFile(cacheFile, 'utf8'));
|
||||
if (cached._cacheVersion === 4 && Date.now() - stat.mtimeMs < (options.maxAgeMs || 30 * 86400000)) return cached.metadata;
|
||||
} catch {}
|
||||
|
||||
const http = new SlowHttp({ minMs: 0, maxMs: 0, maxRetries: 1 });
|
||||
const response = await fetch('https://graphql.anilist.co', {
|
||||
method: 'POST', headers: { 'content-type': 'application/json', 'user-agent': http.userAgent },
|
||||
body: JSON.stringify({ query, variables }),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!response.ok) throw Object.assign(new Error(`AniList HTTP ${response.status}`), { status: response.status });
|
||||
const body = await response.json();
|
||||
if (body.errors?.length) throw new Error(body.errors[0].message);
|
||||
const metadata = metadataFromMedia(body.data.Media);
|
||||
const temporary = `${cacheFile}.${crypto.randomUUID()}.tmp`;
|
||||
try {
|
||||
await fs.writeFile(temporary, JSON.stringify({ _cacheVersion: 4, metadata }, null, 2) + '\n');
|
||||
await fs.rename(temporary, cacheFile);
|
||||
} finally { await fs.rm(temporary, { force: true }); }
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async function lookupAniList(search, options = {}) {
|
||||
const key = Buffer.from(search).toString('base64url');
|
||||
return lookup(QUERY, { search }, key, options);
|
||||
}
|
||||
|
||||
async function lookupAniListById(id, options = {}) {
|
||||
const value = Number(id);
|
||||
if (!Number.isInteger(value) || value < 1) throw new Error('AniList id must be a positive integer');
|
||||
return lookup(QUERY_BY_ID, { id: value }, `id-${value}`, options);
|
||||
}
|
||||
|
||||
module.exports = { lookupAniList, lookupAniListById, metadataFromMedia };
|
||||
@@ -0,0 +1,812 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { adapterFor, adapters } = require('./adapters');
|
||||
const { ArchiveControlError } = require('./lib/control');
|
||||
const { sleep } = require('./lib/delay');
|
||||
const { normalizeScope, selectByScope, scopeLabel } = require('./lib/scope');
|
||||
const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, summarizeInventory, setAutoUpdate } = require('./lib/library-inventory');
|
||||
const { inspectLegacyWork, promoteLegacyWork } = require('./lib/legacy-publication');
|
||||
const { classifyFailure, safeMessage, progressKey } = require('./lib/failures');
|
||||
const { writeJsonAtomic } = require('./lib/json-file');
|
||||
|
||||
const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs'];
|
||||
const FAST_RATE_SETTINGS = {
|
||||
minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000,
|
||||
};
|
||||
const DEFAULT_SOURCE_RATES = {
|
||||
weebcentral: { ...FAST_RATE_SETTINGS },
|
||||
mangadex: { ...FAST_RATE_SETTINGS },
|
||||
};
|
||||
const DEFAULT_SETTINGS = {
|
||||
...FAST_RATE_SETTINGS,
|
||||
sourceRates: Object.fromEntries(Object.entries(DEFAULT_SOURCE_RATES).map(([source, rate]) => [source, { ...rate }])),
|
||||
autoUpdateEnabled: true, autoUpdateTime: '03:00',
|
||||
};
|
||||
|
||||
async function exists(file) {
|
||||
try { await fs.access(file); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
function validateRateSettings(input, prior, prefix = '') {
|
||||
const next = { ...prior };
|
||||
for (const key of RATE_KEYS) {
|
||||
if (input[key] === undefined) continue;
|
||||
const value = Number(input[key]);
|
||||
if (!Number.isFinite(value) || value < 500 || value > 3600000) throw new Error(`${prefix}${key} must be between 500 and 3600000 ms`);
|
||||
next[key] = Math.round(value);
|
||||
}
|
||||
if (next.maxMs < next.minMs) throw new Error(`${prefix}maxMs must be at least ${prefix}minMs`);
|
||||
if (next.chapterMaxMs < next.chapterMinMs) throw new Error(`${prefix}chapterMaxMs must be at least ${prefix}chapterMinMs`);
|
||||
return next;
|
||||
}
|
||||
|
||||
function validateSettings(input = {}, prior = DEFAULT_SETTINGS) {
|
||||
const priorBase = { ...FAST_RATE_SETTINGS, ...prior };
|
||||
const next = validateRateSettings(input, priorBase);
|
||||
if (input.sourceRates !== undefined && (!input.sourceRates || typeof input.sourceRates !== 'object' || Array.isArray(input.sourceRates))) {
|
||||
throw new Error('sourceRates must be an object');
|
||||
}
|
||||
const requestedRates = input.sourceRates || {};
|
||||
const sourceIds = new Set([
|
||||
...Object.keys(DEFAULT_SOURCE_RATES),
|
||||
...Object.keys(prior.sourceRates || {}),
|
||||
...Object.keys(requestedRates),
|
||||
]);
|
||||
next.sourceRates = {};
|
||||
for (const source of sourceIds) {
|
||||
if (!/^[a-z0-9][a-z0-9-]*$/.test(source)) throw new Error(`Invalid source id: ${source}`);
|
||||
const patch = requestedRates[source];
|
||||
if (patch !== undefined && (!patch || typeof patch !== 'object' || Array.isArray(patch))) {
|
||||
throw new Error(`sourceRates.${source} must be an object`);
|
||||
}
|
||||
const base = {
|
||||
...FAST_RATE_SETTINGS,
|
||||
...(DEFAULT_SOURCE_RATES[source] || {}),
|
||||
...((prior.sourceRates || {})[source] || {}),
|
||||
};
|
||||
next.sourceRates[source] = validateRateSettings(patch || {}, base, `sourceRates.${source}.`);
|
||||
}
|
||||
if (input.autoUpdateEnabled !== undefined) next.autoUpdateEnabled = !!input.autoUpdateEnabled;
|
||||
if (input.autoUpdateTime !== undefined) {
|
||||
const value = String(input.autoUpdateTime).trim();
|
||||
if (!/^(?:[01]\d|2[0-3]):[0-5]\d$/.test(value)) throw new Error('autoUpdateTime must be HH:MM');
|
||||
next.autoUpdateTime = value;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function settingsForSource(settings, source) {
|
||||
return { ...settings, ...((settings.sourceRates || {})[source] || {}) };
|
||||
}
|
||||
|
||||
function existingPolicy(value) {
|
||||
const policy = String(value || 'skip-any');
|
||||
if (!['skip-any', 'add-source'].includes(policy)) throw new Error('existingPolicy must be skip-any or add-source');
|
||||
return policy;
|
||||
}
|
||||
|
||||
function nextSchedule(time, from = new Date()) {
|
||||
const [hours, minutes] = String(time).split(':').map(Number);
|
||||
const next = new Date(from);
|
||||
next.setHours(hours, minutes, 0, 0);
|
||||
if (next <= from) next.setDate(next.getDate() + 1);
|
||||
return next;
|
||||
}
|
||||
|
||||
class ScraperService {
|
||||
constructor(options = {}) {
|
||||
this.libraryRoot = path.resolve(options.libraryRoot || 'library');
|
||||
this.dataRoot = path.resolve(options.dataRoot || path.join('data', 'scraper'));
|
||||
this.stateFile = path.join(this.dataRoot, 'queue.json');
|
||||
this.logFile = path.join(this.dataRoot, 'events.log');
|
||||
this.lockFile = path.join(this.dataRoot, 'daemon.lock');
|
||||
this.legacyStateFile = path.join(this.libraryRoot, '.mcscraper-queue', 'queue.json');
|
||||
this.state = null;
|
||||
this.stopping = false;
|
||||
this.loopPromise = null;
|
||||
this.active = new Map();
|
||||
this.writeChain = Promise.resolve();
|
||||
this.discoveryTimer = null;
|
||||
this.inventoryCache = null;
|
||||
this.inventoryCachedAt = 0;
|
||||
this.inventoryGeneration = 0;
|
||||
this.inventoryCacheGeneration = -1;
|
||||
this.inventoryRefresh = null;
|
||||
this.inventoryAttemptedAt = 0;
|
||||
this.inventoryError = null;
|
||||
this.scanInventory = options.scanLibrary || scanLibrary;
|
||||
this.resolveAdapter = options.adapterFor || adapterFor;
|
||||
this.adapterList = options.adapters || adapters;
|
||||
this.retryDelayMs = options.retryDelayMs ?? 300000;
|
||||
this.runtimeHeartbeat = Date.now();
|
||||
}
|
||||
|
||||
async init() {
|
||||
await fs.mkdir(this.dataRoot, { recursive: true });
|
||||
await fs.mkdir(this.libraryRoot, { recursive: true });
|
||||
await this.acquireLock();
|
||||
this.state = await this.loadState();
|
||||
this.state.incidents = this.state.incidents || [];
|
||||
this.state.circuits = this.state.circuits || {};
|
||||
this.state.updateChecks = this.state.updateChecks || {};
|
||||
// Older releases retained successful no-change checks in the work queue.
|
||||
// Keep their latest result before retiring those finished rows.
|
||||
for (const item of this.state.queue) {
|
||||
if (item.kind !== 'discovery' || item.status !== 'complete' || !item.checked || !item.libraryMangaId) continue;
|
||||
const key = JSON.stringify([item.libraryMangaId, item.source]);
|
||||
const prior = this.state.updateChecks[key];
|
||||
if (!prior?.lastCheckedAt || Date.parse(item.checked) > Date.parse(prior.lastCheckedAt)) {
|
||||
this.state.updateChecks[key] = { ...prior, mangaId: item.libraryMangaId, source: item.source,
|
||||
lastCheckedAt: item.checked, lastResult: 'no-updates' };
|
||||
}
|
||||
}
|
||||
this.state.queue = this.state.queue.filter(item => item.kind !== 'discovery' || item.status !== 'complete' || !item.libraryMangaId);
|
||||
this.state.queue.forEach((item, index) => { item.order = index; });
|
||||
for (const item of this.state.queue) {
|
||||
delete item.stopReason;
|
||||
if (item.status === 'downloading' || item.status === 'paused' || item.status === 'discovering') item.status = 'queued';
|
||||
const found = this.resolveAdapter(item.url);
|
||||
item.source = found?.adapter.id || item.source || 'unknown';
|
||||
item.seriesId = item.seriesId || found?.info.id || item.id;
|
||||
item.kind = ['discovery', 'inspection'].includes(item.kind) ? item.kind : 'download';
|
||||
item.scope = normalizeScope(item.scope || {});
|
||||
item.scopeLabel = item.scopeLabel || scopeLabel(item.scope);
|
||||
item.existingPolicy = existingPolicy(item.existingPolicy);
|
||||
if (item.status === 'error') {
|
||||
item.failure = item.failure || classifyFailure(new Error(item.error));
|
||||
if (item.failure.kind === 'unavailable') item.status = 'unavailable';
|
||||
}
|
||||
if (item.source === 'mangadex' && item.status === 'unavailable' && !item.availability
|
||||
&& item.error === 'No downloadable English MangaDex chapters found') {
|
||||
item.error = 'The previous adapter selected no numbered hosted English chapters. Inspect this source again to verify its edition and availability, including pilots and specials.';
|
||||
if (item.failure) item.failure = { ...item.failure, message: item.error };
|
||||
}
|
||||
}
|
||||
this.state.version = 4;
|
||||
this.state.currentId = null;
|
||||
this.state.currentIds = {};
|
||||
this.state.sourcePaused = this.state.sourcePaused || {};
|
||||
// A legacy title-unavailable pause can be released; access and unknown failures remain paused.
|
||||
for (const source of Object.keys(this.state.sourcePaused)) {
|
||||
const failed = this.state.queue.filter(item => item.source === source && ['error', 'unavailable', 'quarantined'].includes(item.status));
|
||||
if (failed.length && failed.every(item => item.failure?.kind === 'unavailable')) delete this.state.sourcePaused[source];
|
||||
}
|
||||
this.state.status = this.state.paused ? 'paused' : 'idle';
|
||||
this.state.pid = process.pid;
|
||||
await this.event('info', 'Scraper daemon started', { pid: process.pid });
|
||||
this.scheduleDiscovery();
|
||||
this.loopPromise = this.workLoop();
|
||||
return this;
|
||||
}
|
||||
|
||||
async acquireLock() {
|
||||
try {
|
||||
const handle = await fs.open(this.lockFile, 'wx');
|
||||
await handle.writeFile(String(process.pid));
|
||||
return await handle.close();
|
||||
} catch (error) {
|
||||
if (error.code !== 'EEXIST') throw error;
|
||||
}
|
||||
const oldPid = Number((await fs.readFile(this.lockFile, 'utf8').catch(() => '')).trim());
|
||||
try {
|
||||
if (oldPid) process.kill(oldPid, 0);
|
||||
throw new Error(`Scraper daemon is already running (PID ${oldPid || 'unknown'})`);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ESRCH') throw error;
|
||||
await fs.rm(this.lockFile, { force: true });
|
||||
const handle = await fs.open(this.lockFile, 'wx');
|
||||
await handle.writeFile(String(process.pid));
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async loadState() {
|
||||
if (await exists(this.stateFile)) {
|
||||
const saved = JSON.parse(await fs.readFile(this.stateFile, 'utf8'));
|
||||
return {
|
||||
version: 3, paused: false, status: 'idle', queue: [], history: [], publishedRevision: 0,
|
||||
currentIds: {}, sourcePaused: {},
|
||||
settings: DEFAULT_SETTINGS, ...saved,
|
||||
settings: validateSettings(saved.settings || {}, DEFAULT_SETTINGS),
|
||||
};
|
||||
}
|
||||
const state = {
|
||||
version: 3, paused: false, status: 'idle', currentId: null, currentIds: {}, sourcePaused: {},
|
||||
settings: { ...DEFAULT_SETTINGS }, queue: [], history: [], publishedRevision: 0,
|
||||
created: new Date().toISOString(), lastError: null,
|
||||
};
|
||||
if (await exists(this.legacyStateFile)) {
|
||||
const legacy = JSON.parse(await fs.readFile(this.legacyStateFile, 'utf8'));
|
||||
const completed = new Set(legacy.completed || []);
|
||||
state.queue = (legacy.entries || []).map((item, index) => ({
|
||||
id: item.id, seriesId: item.id, source: 'weebcentral', title: item.title, url: item.url,
|
||||
metadataSearch: item.metadataSearch || item.title,
|
||||
kind: 'download', scope: { mode: 'missing' }, existingPolicy: 'skip-any',
|
||||
status: completed.has(item.id) ? 'complete' : 'queued', order: index,
|
||||
added: legacy.created || new Date().toISOString(),
|
||||
}));
|
||||
state.settings = validateSettings(legacy.settings || {}, DEFAULT_SETTINGS);
|
||||
state.migratedFrom = this.legacyStateFile;
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
async persist() {
|
||||
const write = async () => {
|
||||
this.state.updated = new Date().toISOString();
|
||||
await writeJsonAtomic(this.stateFile, this.state);
|
||||
};
|
||||
const pending = this.writeChain.then(write, write);
|
||||
this.writeChain = pending.catch(() => {});
|
||||
return pending;
|
||||
}
|
||||
|
||||
async event(level, message, details = {}) {
|
||||
const entry = { time: new Date().toISOString(), level, message: safeMessage(message), ...details };
|
||||
if (entry.url) entry.url = safeMessage(entry.url);
|
||||
await fs.appendFile(this.logFile, JSON.stringify(entry) + '\n');
|
||||
console.log(JSON.stringify({ service: 'scraper', ...entry }));
|
||||
this.state.lastEvent = entry;
|
||||
if (level === 'error') this.state.lastError = entry;
|
||||
await this.persist();
|
||||
}
|
||||
|
||||
async recentEvents(limit = 30) {
|
||||
const handle = await fs.open(this.logFile, 'r').catch(error => {
|
||||
if (error.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
});
|
||||
if (!handle) return [];
|
||||
let text;
|
||||
try {
|
||||
const { size } = await handle.stat();
|
||||
const length = Math.min(size, 512 * 1024), start = size - length;
|
||||
const buffer = Buffer.alloc(length);
|
||||
const { bytesRead } = await handle.read(buffer, 0, length, start);
|
||||
text = buffer.subarray(0, bytesRead).toString('utf8');
|
||||
if (start > 0) {
|
||||
const boundary = text.indexOf('\n');
|
||||
text = boundary < 0 ? '' : text.slice(boundary + 1);
|
||||
}
|
||||
} finally { await handle.close(); }
|
||||
return text.trim().split(/\r?\n/).filter(Boolean).slice(-Math.max(1, Math.min(200, limit)))
|
||||
.map(line => { try { return JSON.parse(line); } catch { return { level: 'warning', message: line }; } });
|
||||
}
|
||||
|
||||
invalidateInventory() {
|
||||
// Keep the last summary visible while rebuilding. Download/discovery decisions
|
||||
// still await an authoritative scan through libraryInventory(), never this stale view.
|
||||
this.inventoryGeneration++;
|
||||
}
|
||||
|
||||
refreshInventory() {
|
||||
if (this.inventoryRefresh) return this.inventoryRefresh;
|
||||
const generation = this.inventoryGeneration;
|
||||
this.inventoryAttemptedAt = Date.now();
|
||||
this.inventoryRefresh = Promise.resolve().then(() => this.scanInventory(this.libraryRoot))
|
||||
.then(inventory => {
|
||||
// A chapter or metadata edit during the scan invalidates its result.
|
||||
if (generation === this.inventoryGeneration) {
|
||||
this.inventoryCache = inventory;
|
||||
this.inventoryCachedAt = Date.now();
|
||||
this.inventoryCacheGeneration = generation;
|
||||
}
|
||||
this.inventoryError = null;
|
||||
return inventory;
|
||||
}).catch(error => {
|
||||
this.inventoryError = safeMessage(error.message);
|
||||
throw error;
|
||||
}).finally(() => { this.inventoryRefresh = null; });
|
||||
return this.inventoryRefresh;
|
||||
}
|
||||
|
||||
async libraryInventory(force = false) {
|
||||
if (!force && this.inventoryCache && this.inventoryCacheGeneration === this.inventoryGeneration
|
||||
&& Date.now() - this.inventoryCachedAt < 60000) return this.inventoryCache;
|
||||
do {
|
||||
await this.refreshInventory();
|
||||
} while (this.inventoryCacheGeneration !== this.inventoryGeneration);
|
||||
return this.inventoryCache;
|
||||
}
|
||||
|
||||
scheduleDiscovery() {
|
||||
if (this.discoveryTimer) clearTimeout(this.discoveryTimer);
|
||||
this.discoveryTimer = null;
|
||||
if (!this.state?.settings?.autoUpdateEnabled || this.stopping) {
|
||||
if (this.state) this.state.nextDiscoveryAt = null;
|
||||
return;
|
||||
}
|
||||
const next = nextSchedule(this.state.settings.autoUpdateTime);
|
||||
this.state.nextDiscoveryAt = next.toISOString();
|
||||
const delay = Math.max(1000, next.getTime() - Date.now());
|
||||
this.discoveryTimer = setTimeout(() => {
|
||||
this.discoveryTimer = null;
|
||||
this.enqueueDiscoveryChecks('schedule').catch(error => this.event('error', `Nightly discovery failed: ${error.message}`))
|
||||
.finally(() => { if (!this.stopping) this.scheduleDiscovery(); });
|
||||
}, delay);
|
||||
this.discoveryTimer.unref();
|
||||
}
|
||||
|
||||
pendingSeries(source, seriesId) {
|
||||
return this.state.queue.some(item => item.source === source && String(item.seriesId) === String(seriesId)
|
||||
&& ['queued', 'downloading', 'discovering', 'ready', 'error', 'retry_wait', 'quarantined', 'unavailable'].includes(item.status));
|
||||
}
|
||||
|
||||
recordUpdateCheck(item, patch) {
|
||||
if (!item.libraryMangaId || !item.source) return;
|
||||
const key = JSON.stringify([item.libraryMangaId, item.source]);
|
||||
this.state.updateChecks[key] = { ...(this.state.updateChecks[key] || {}),
|
||||
mangaId: item.libraryMangaId, source: item.source, ...patch };
|
||||
}
|
||||
|
||||
async enqueueDiscoveryChecks(reason = 'manual') {
|
||||
const inventory = await this.libraryInventory(true);
|
||||
let queued = 0, skipped = 0;
|
||||
for (const manga of inventory.mangas) {
|
||||
if (!manga.autoUpdate) continue;
|
||||
const supported = manga.sources.map(source => ({ source, found: this.resolveAdapter(source.url) })).filter(item => item.found);
|
||||
if (!supported.length) { skipped++; continue; }
|
||||
const preferred = manga.meta?.['x-scraper']?.preferredSource;
|
||||
const selected = supported.find(item => item.found.adapter.id === preferred) || supported[0];
|
||||
const source = selected.found.adapter.id, seriesId = selected.found.info.id;
|
||||
if (this.pendingSeries(source, seriesId)) { skipped++; continue; }
|
||||
const item = {
|
||||
id: crypto.randomUUID(), seriesId, source, kind: 'discovery',
|
||||
title: manga.title, url: selected.found.info.url, metadataSearch: manga.title,
|
||||
libraryMangaId: manga.id, scope: { mode: 'missing' }, existingPolicy: 'skip-any',
|
||||
status: 'queued', order: this.state.queue.length, added: new Date().toISOString(), reason,
|
||||
};
|
||||
this.state.queue.push(item); queued++;
|
||||
}
|
||||
this.state.lastDiscovery = { time: new Date().toISOString(), reason, queued, skipped };
|
||||
await this.event('info', `Queued ${queued} update check${queued === 1 ? '' : 's'}`, { reason, queued, skipped });
|
||||
return { queued, skipped };
|
||||
}
|
||||
|
||||
async setMangaAutoUpdate(id, enabled) {
|
||||
const result = await setAutoUpdate(this.libraryRoot, id, enabled);
|
||||
this.invalidateInventory();
|
||||
await this.event('info', `${enabled ? 'Enabled' : 'Disabled'} auto update for ${id}`, { mangaId: id, enabled: !!enabled });
|
||||
return result;
|
||||
}
|
||||
|
||||
async publishLegacyReady() {
|
||||
if (!this.state.paused || this.active.size || Object.keys(this.state.currentIds || {}).length) {
|
||||
throw new Error('Pause the queue and wait for active workers to stop before publishing legacy work');
|
||||
}
|
||||
const candidates = [];
|
||||
const seen = new Set();
|
||||
for (const item of this.state.queue) {
|
||||
const seriesId = String(item.seriesId || item.id || '');
|
||||
if (!seriesId || seen.has(seriesId) || item.status === 'complete') continue;
|
||||
seen.add(seriesId);
|
||||
const inspection = await inspectLegacyWork({ libraryRoot: this.libraryRoot, seriesId });
|
||||
if (inspection) candidates.push({ item, inspection });
|
||||
}
|
||||
const published = [];
|
||||
for (const candidate of candidates) {
|
||||
const result = await promoteLegacyWork(candidate.inspection);
|
||||
candidate.item.libraryMangaId = result.mangaId;
|
||||
candidate.item.legacyPromotedAt = new Date().toISOString();
|
||||
published.push(result);
|
||||
this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1;
|
||||
this.invalidateInventory();
|
||||
await this.event('info', `Published ${result.chaptersPublished} completed chapters for ${candidate.item.title}`, {
|
||||
id: candidate.item.id, seriesId: candidate.item.seriesId, title: candidate.item.title,
|
||||
mangaId: result.mangaId, chaptersPublished: result.chaptersPublished,
|
||||
});
|
||||
}
|
||||
return { published };
|
||||
}
|
||||
|
||||
async runDiscovery(item, found, source) {
|
||||
if (this.stopping || this.state.paused) throw new ArchiveControlError(this.stopping ? 'stopped' : 'paused');
|
||||
item.status = 'discovering';
|
||||
const catalog = await found.adapter.discover(item, {
|
||||
settings: settingsForSource(this.state.settings, source),
|
||||
shouldStop: () => this.stopping ? 'stopped' : this.state.paused ? 'paused' : item.stopReason || false,
|
||||
onEvent: event => this.event(event.level, event.message, { id: item.id, title: item.title, source, ...(event.status ? { status: event.status } : {}) }),
|
||||
});
|
||||
if (this.stopping || this.state.paused || item.stopReason) throw new ArchiveControlError(item.stopReason || (this.stopping ? 'stopped' : 'paused'));
|
||||
await this.recordCatalog(item, catalog);
|
||||
if (item.kind === 'inspection') {
|
||||
if (!catalog.chapters?.length) throw Object.assign(new Error(catalog.availability?.message || 'This source edition has no downloadable chapters'), {
|
||||
failureKind: catalog.availability?.reason === 'unrecognized-labels' ? 'adapter' : 'unavailable',
|
||||
});
|
||||
if (item.inspectionFromComplete) {
|
||||
const inventory = await this.libraryInventory(true);
|
||||
const manga = findManga(inventory, {
|
||||
source, seriesId: item.seriesId, sourceUrl: item.url,
|
||||
anilist: catalog.links?.al, libraryMangaId: item.libraryMangaId,
|
||||
});
|
||||
const missing = selectByScope(catalog.chapters, item.scope)
|
||||
.filter(ref => shouldDownload(manga, ref, source, item.existingPolicy));
|
||||
if (!missing.length) {
|
||||
item.kind = 'download'; item.status = 'complete'; item.checked = new Date().toISOString();
|
||||
delete item.inspectionFromComplete;
|
||||
for (const incident of this.state.incidents) if (incident.jobId === item.id && incident.state === 'open') {
|
||||
incident.state = 'resolved'; incident.resolution = 'inspected-complete';
|
||||
}
|
||||
await this.event('info', `Source inspected: ${item.sourceTitle}; completed download is already in the library`, { id: item.id, source });
|
||||
return;
|
||||
}
|
||||
}
|
||||
delete item.inspectionFromComplete;
|
||||
item.status = 'ready'; item.checked = new Date().toISOString();
|
||||
for (const incident of this.state.incidents) if (incident.jobId === item.id && incident.state === 'open') {
|
||||
incident.state = 'resolved'; incident.resolution = 'inspected-ready';
|
||||
}
|
||||
await this.event('info', `Source inspected: ${item.sourceTitle}; awaiting download confirmation`, { id: item.id, source });
|
||||
return;
|
||||
}
|
||||
const inventory = await this.libraryInventory(true);
|
||||
if (this.stopping || this.state.paused) throw new ArchiveControlError(this.stopping ? 'stopped' : 'paused');
|
||||
const manga = findManga(inventory, {
|
||||
source, seriesId: item.seriesId, sourceUrl: item.url,
|
||||
anilist: catalog.links?.al, libraryMangaId: item.libraryMangaId,
|
||||
});
|
||||
if (item.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${item.libraryMangaId}`);
|
||||
const candidates = selectByScope(catalog.chapters || [], item.scope)
|
||||
.filter(ref => shouldDownload(manga, ref, source, item.existingPolicy))
|
||||
.filter(ref => isNewerThanLibrary(manga, ref));
|
||||
item.checked = new Date().toISOString();
|
||||
item.catalogTotal = catalog.chapters?.length || 0;
|
||||
this.recordUpdateCheck(item, { lastCheckedAt: item.checked,
|
||||
...(candidates.length ? { lastNewContentAt: item.checked, lastResult: 'new-content' } : { lastResult: 'no-updates' }) });
|
||||
if (!candidates.length) {
|
||||
item.status = 'complete'; item.completed = item.checked; item.discovered = 0;
|
||||
this.state.history.push({ id: item.id, title: item.title, source, status: 'checked', time: item.completed, discovered: 0 });
|
||||
this.state.queue = this.state.queue.filter(entry => entry.id !== item.id);
|
||||
this.state.queue.forEach((entry, index) => { entry.order = index; });
|
||||
await this.event('info', `No updates found for ${item.title}`, { id: item.id, title: item.title, source });
|
||||
return;
|
||||
}
|
||||
item.kind = 'download'; item.status = 'queued'; item.discovered = candidates.length;
|
||||
item.scope = normalizeScope({ mode: 'ids', ids: candidates.map(ref => ref.id) });
|
||||
item.scopeLabel = scopeLabel(item.scope);
|
||||
await this.event('info', `Found ${candidates.length} new chapter${candidates.length === 1 ? '' : 's'} for ${item.title}`, {
|
||||
id: item.id, title: item.title, source, discovered: candidates.length,
|
||||
});
|
||||
}
|
||||
|
||||
async recordCatalog(item, catalog) {
|
||||
const title = String(catalog.title || item.title).slice(0, 200);
|
||||
const changedEdition = item.confirmedSourceTitle && item.confirmedSourceTitle !== title;
|
||||
item.sourceTitle = title;
|
||||
item.catalogTotal = catalog.chapters?.length || 0;
|
||||
item.availability = catalog.availability || { selectedChapters: item.catalogTotal, coverage: 'unknown' };
|
||||
await this.persist();
|
||||
if (changedEdition) throw Object.assign(new Error('The source edition title changed since inspection. Inspect and confirm it again before downloading.'), { failureKind: 'unavailable' });
|
||||
}
|
||||
|
||||
async jobState(item) {
|
||||
if (!item) return null;
|
||||
if (item.kind === 'inspection') return { status: 'discovering', updated: item.started, completed: 0, total: null, current: null };
|
||||
const file = path.join(this.libraryRoot, `.mcscraper-work-${item.seriesId || item.id}`, 'job.json');
|
||||
try {
|
||||
const job = JSON.parse(await fs.readFile(file, 'utf8'));
|
||||
return {
|
||||
status: job.status || 'unknown', completed: job.publicationMode === 'incremental' ? Number(job.jobCompleted || 0) : job.completed?.length || 0, total: job.total || null,
|
||||
percent: job.total ? Math.round((job.publicationMode === 'incremental' ? Number(job.jobCompleted || 0) : job.completed?.length || 0) * 1000 / job.total) / 10 : null,
|
||||
lastChapter: job.lastChapter || null, current: job.current || null,
|
||||
lastProblem: job.lastProblem || null, lastError: job.lastError || null,
|
||||
problemCount: job.problemCount || 0, rate: job.rate || null, updated: job.updated || null,
|
||||
};
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
async publicState() {
|
||||
const activeItems = this.state.queue.filter(item => Object.values(this.state.currentIds || {}).includes(item.id));
|
||||
const active = await Promise.all(activeItems.map(async current => ({
|
||||
source: this.resolveAdapter(current.url)?.adapter.id || current.source,
|
||||
current, job: await this.jobState(current),
|
||||
})));
|
||||
const current = active[0]?.current || null;
|
||||
const job = active[0]?.job || null;
|
||||
// Status and watchdog probes must not wait for thousands of bind-mount reads.
|
||||
// Share one background scan, serve the previous summary, and surface its age/error.
|
||||
if (!this.inventoryRefresh && (this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000)
|
||||
&& Date.now() - this.inventoryAttemptedAt >= 10000) {
|
||||
this.refreshInventory().catch(() => {});
|
||||
}
|
||||
const inventory = this.inventoryCache;
|
||||
return {
|
||||
version: this.state.version, status: this.state.status, paused: this.state.paused,
|
||||
pid: this.state.pid, settings: this.state.settings, publishedRevision: this.state.publishedRevision || 0,
|
||||
current, job, active, sourcePaused: this.state.sourcePaused || {}, queue: this.state.queue,
|
||||
history: (this.state.history || []).slice(-20).reverse(), lastError: this.state.lastError || null,
|
||||
lastEvent: this.state.lastEvent || null, updated: this.state.updated || null,
|
||||
nextDiscoveryAt: this.state.nextDiscoveryAt || null, lastDiscovery: this.state.lastDiscovery || null,
|
||||
library: inventory ? summarizeInventory(inventory, value => this.resolveAdapter(value)) : [],
|
||||
updateChecks: Object.values(this.state.updateChecks || {}),
|
||||
libraryStatus: {
|
||||
ready: !!inventory, refreshing: !!this.inventoryRefresh,
|
||||
stale: this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000,
|
||||
updated: this.inventoryCachedAt ? new Date(this.inventoryCachedAt).toISOString() : null,
|
||||
error: this.inventoryError,
|
||||
},
|
||||
adapters: this.adapterList.map(item => ({ id: item.id, label: item.label })),
|
||||
events: await this.recentEvents(25),
|
||||
incidents: (this.state.incidents || []).slice(-50).reverse(), circuits: this.state.circuits || {},
|
||||
runtimeHeartbeat: new Date(this.runtimeHeartbeat).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
async enqueue(input) {
|
||||
const found = this.resolveAdapter(input.url);
|
||||
if (!found) throw new Error('No installed scraper adapter accepts this URL');
|
||||
const scope = normalizeScope(input.scope || {});
|
||||
const policy = existingPolicy(input.existingPolicy);
|
||||
const signature = JSON.stringify([found.adapter.id, found.info.id, scope, policy, input.libraryMangaId || null]);
|
||||
if (this.state.queue.some(item => item.status !== 'complete'
|
||||
&& JSON.stringify([item.source, item.seriesId, item.scope, item.existingPolicy, item.libraryMangaId || null]) === signature)) {
|
||||
throw new Error('That exact manga scope is already queued');
|
||||
}
|
||||
const item = {
|
||||
...found.info, id: crypto.randomUUID(), seriesId: found.info.id, kind: input.inspect === true ? 'inspection' : 'download',
|
||||
title: String(input.title || found.info.title).trim().slice(0, 200),
|
||||
metadataSearch: String(input.metadataSearch || input.title || found.info.title).trim().slice(0, 200),
|
||||
scope, scopeLabel: scopeLabel(scope), existingPolicy: policy,
|
||||
...(input.libraryMangaId ? { libraryMangaId: String(input.libraryMangaId) } : {}),
|
||||
status: 'queued', order: this.state.queue.length, added: new Date().toISOString(),
|
||||
};
|
||||
this.state.queue.push(item);
|
||||
await this.event('info', `Queued ${item.title}: ${item.scopeLabel}`, { id: item.id, seriesId: item.seriesId, title: item.title });
|
||||
return item;
|
||||
}
|
||||
|
||||
async remove(id) {
|
||||
const item = this.state.queue.find(entry => entry.id === id);
|
||||
if (!item) throw new Error('Queue item not found');
|
||||
if (Object.values(this.state.currentIds || {}).includes(item.id)) throw new Error('Pause the active download before removing it');
|
||||
this.state.queue = this.state.queue.filter(entry => entry.id !== id);
|
||||
this.state.queue.forEach((entry, index) => { entry.order = index; });
|
||||
await this.event('info', `Removed ${item.title} from queue`, { id: item.id, title: item.title });
|
||||
}
|
||||
|
||||
async clearCompleted(source = '') {
|
||||
if (source && !this.adapterList.some(adapter => adapter.id === source)) throw new Error('Unknown source');
|
||||
const before = this.state.queue.length;
|
||||
const activeIds = new Set(Object.values(this.state.currentIds || {}));
|
||||
this.state.queue = this.state.queue.filter(item => !(item.status === 'complete' && item.kind === 'download'
|
||||
&& !activeIds.has(item.id) && (!source || item.source === source)));
|
||||
this.state.queue.forEach((item, index) => { item.order = index; });
|
||||
const cleared = before - this.state.queue.length;
|
||||
if (cleared) await this.event('info', `Cleared ${cleared} completed download job${cleared === 1 ? '' : 's'} from queue`, { source: source || 'all', cleared });
|
||||
return { cleared };
|
||||
}
|
||||
|
||||
async reorder(ids) {
|
||||
if (!Array.isArray(ids) || ids.length !== this.state.queue.length || new Set(ids).size !== ids.length) throw new Error('Order must contain every queue id exactly once');
|
||||
const byId = new Map(this.state.queue.map(item => [item.id, item]));
|
||||
if (ids.some(id => !byId.has(id))) throw new Error('Order contains an unknown queue id');
|
||||
for (const current of Object.values(this.state.currentIds || {})) {
|
||||
if (ids.indexOf(current) !== this.state.queue.findIndex(item => item.id === current)) throw new Error('Active items cannot be moved while downloading');
|
||||
}
|
||||
this.state.queue = ids.map((id, index) => ({ ...byId.get(id), order: index }));
|
||||
await this.event('info', 'Queue reordered');
|
||||
}
|
||||
|
||||
async updateSettings(patch) {
|
||||
this.state.settings = validateSettings(patch, this.state.settings);
|
||||
this.scheduleDiscovery();
|
||||
await this.event('info', 'Scraper settings updated', { settings: this.state.settings, nextDiscoveryAt: this.state.nextDiscoveryAt });
|
||||
return this.state.settings;
|
||||
}
|
||||
|
||||
async control(action, id) {
|
||||
if (action === 'inspect' || action === 'confirm') {
|
||||
const item = this.state.queue.find(entry => entry.id === id);
|
||||
if (!item || Object.values(this.state.currentIds || {}).includes(item.id)) throw new Error('Job not found or already active');
|
||||
if (this.state.sourcePaused[item.source] || item.failure?.kind === 'access') throw new Error('Review the source access block before inspecting or confirming');
|
||||
if (action === 'confirm') {
|
||||
if (item.status !== 'ready' || item.kind !== 'inspection') throw new Error('Inspect the source first');
|
||||
item.confirmedSourceTitle = item.sourceTitle;
|
||||
item.kind = 'download';
|
||||
} else {
|
||||
if (!['unavailable', 'quarantined', 'complete', 'ready'].includes(item.status)) throw new Error('Only inactive finished, unavailable or quarantined jobs can be inspected again');
|
||||
item.inspectionFromComplete = !!item.completed && Number(item.publishedCount || 0) > 0;
|
||||
item.kind = 'inspection'; delete item.confirmedSourceTitle;
|
||||
}
|
||||
item.status = 'queued'; delete item.error; delete item.failure; delete item.nextAttemptAt;
|
||||
await this.event('info', action === 'confirm' ? `Download confirmed: ${item.sourceTitle}` : `Source inspection queued: ${item.title}`, { id: item.id, source: item.source });
|
||||
return;
|
||||
}
|
||||
if (action === 'pause') {
|
||||
this.state.paused = true;
|
||||
this.state.status = 'pausing';
|
||||
await this.event('info', 'Pause requested');
|
||||
return;
|
||||
}
|
||||
if (action === 'resume') {
|
||||
this.state.paused = false;
|
||||
this.recomputeStatus();
|
||||
await this.event('info', 'Queue resumed');
|
||||
return;
|
||||
}
|
||||
if (action === 'retry') {
|
||||
const item = this.state.queue.find(entry => entry.id === id) || this.state.queue.find(entry => entry.status === 'error');
|
||||
if (!item) throw new Error('No failed queue item to retry');
|
||||
if (Object.values(this.state.currentIds || {}).includes(item.id)) throw new Error('That item is already active');
|
||||
const source = this.resolveAdapter(item.url)?.adapter.id || item.source;
|
||||
const otherAccessError = this.state.queue.some(entry => entry.id !== item.id && entry.source === source && entry.status === 'error' && entry.failure?.kind === 'access');
|
||||
if (otherAccessError) throw new Error('Review the access-blocked job for this source first');
|
||||
item.status = 'queued'; delete item.error; delete item.failure; delete item.nextAttemptAt;
|
||||
item.recoveryAttempts = 0;
|
||||
delete this.state.sourcePaused[source]; delete this.state.circuits[source];
|
||||
for (const incident of this.state.incidents) if (incident.jobId === item.id && incident.state === 'open') { incident.state = 'resolved'; incident.resolution = 'operator-retry'; }
|
||||
this.state.lastError = null; this.state.paused = false; this.recomputeStatus();
|
||||
await this.event('info', `Retrying ${item.title}`, { id: item.id, title: item.title });
|
||||
return;
|
||||
}
|
||||
if (action === 'skip') {
|
||||
const item = this.state.queue.find(entry => entry.id === id);
|
||||
if (!item || this.state.currentIds[item.source] === item.id) throw new Error('Job not found or it is active');
|
||||
item.status = 'skipped';
|
||||
if (item.failure?.scope === 'job') delete this.state.sourcePaused[item.source];
|
||||
await this.event('info', `Skipped ${item.title}`, { id: item.id, source: item.source });
|
||||
return;
|
||||
}
|
||||
throw new Error('Unknown control action');
|
||||
}
|
||||
|
||||
async recover(input) {
|
||||
const item = this.state.queue.find(entry => entry.id === input.id);
|
||||
if (!item) throw new Error('Job not found');
|
||||
// Compare observations so a stale watcher cannot disturb a progressing worker.
|
||||
if (Object.values(this.state.currentIds).includes(item.id)) {
|
||||
const job = await this.jobState(item);
|
||||
if (progressKey(job) !== input.progressKey || this.state.paused || item.stopReason) return { changed: false };
|
||||
if (Date.now() - Date.parse(job?.updated || item.started) < 900000) return { changed: false };
|
||||
if ((item.stallRecoveries || 0) >= 1) {
|
||||
item.stopReason = 'quarantined';
|
||||
} else { item.stallRecoveries = (item.stallRecoveries || 0) + 1; item.stopReason = 'recovering'; }
|
||||
await this.event('warning', `Sentinel requested recovery at the next checkpoint for ${item.title}`, { id: item.id, source: item.source });
|
||||
return { changed: true };
|
||||
}
|
||||
return { changed: false };
|
||||
}
|
||||
|
||||
recomputeStatus() {
|
||||
const activeIds = Object.values(this.state.currentIds || {});
|
||||
this.state.currentId = activeIds[0] || null;
|
||||
if (this.stopping) this.state.status = 'stopping';
|
||||
else if (this.state.paused) this.state.status = activeIds.length ? 'pausing' : 'paused';
|
||||
else if (activeIds.length) this.state.status = 'downloading';
|
||||
else if (this.state.queue.some(item => item.status === 'retry_wait')) this.state.status = 'retry_wait';
|
||||
else if (this.state.queue.some(item => ['error', 'quarantined'].includes(item.status))) this.state.status = 'attention';
|
||||
else this.state.status = 'idle';
|
||||
}
|
||||
|
||||
startItem(item, found) {
|
||||
const source = found.adapter.id;
|
||||
item.status = 'downloading'; item.started = item.kind === 'inspection' ? new Date().toISOString() : item.started || new Date().toISOString(); delete item.error;
|
||||
this.state.currentIds[source] = item.id;
|
||||
this.state.lastError = null;
|
||||
this.recomputeStatus();
|
||||
const entry = { id: item.id, source, promise: null };
|
||||
this.active.set(source, entry);
|
||||
entry.promise = this.runItem(item, found, source).finally(async () => {
|
||||
this.active.delete(source);
|
||||
if (this.state.currentIds[source] === item.id) delete this.state.currentIds[source];
|
||||
this.recomputeStatus();
|
||||
await this.persist();
|
||||
}).catch(error => { console.error('[scraper] persistence failed:', safeMessage(error)); this.state.paused = true; });
|
||||
}
|
||||
|
||||
async runItem(item, found, source) {
|
||||
await this.event('info', `${item.kind === 'discovery' ? 'Checking' : 'Starting'} ${item.title}`, { id: item.id, title: item.title, source, kind: item.kind });
|
||||
try {
|
||||
if (item.kind === 'discovery' || item.kind === 'inspection') {
|
||||
await this.runDiscovery(item, found, source);
|
||||
return;
|
||||
}
|
||||
let revisionNotifications = 0;
|
||||
const result = await found.adapter.run(item, {
|
||||
libraryRoot: this.libraryRoot, settings: settingsForSource(this.state.settings, source),
|
||||
shouldStop: () => this.stopping ? 'stopped' : this.state.paused ? 'paused' : item.stopReason || false,
|
||||
onCatalog: catalog => this.recordCatalog(item, catalog),
|
||||
onPublished: async details => {
|
||||
revisionNotifications++;
|
||||
this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1;
|
||||
const mangaId = item.libraryMangaId || (details.mangaDir && path.basename(details.mangaDir));
|
||||
if (mangaId) this.recordUpdateCheck({ ...item, libraryMangaId: mangaId }, { lastDownloadedAt: new Date().toISOString() });
|
||||
this.invalidateInventory();
|
||||
await this.event('info', `Published ${item.title}${details.chapter ? ` chapter ${details.chapter}` : ''}`, {
|
||||
id: item.id, title: item.title, source, ...(details.chapter ? { chapter: details.chapter } : {}),
|
||||
});
|
||||
},
|
||||
});
|
||||
const finalPath = typeof result === 'string' ? result : result?.finalPath || null;
|
||||
if (!revisionNotifications && (typeof result === 'string' || Number(result?.publishedCount || 0) > 0)) {
|
||||
this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1;
|
||||
this.invalidateInventory();
|
||||
}
|
||||
item.status = 'complete'; item.completed = new Date().toISOString(); item.finalPath = finalPath;
|
||||
delete this.state.circuits[source];
|
||||
for (const incident of this.state.incidents) if (incident.jobId === item.id && incident.state === 'open') { incident.state = 'resolved'; incident.resolution = 'recovered'; }
|
||||
item.publishedCount = Number(result?.publishedCount ?? 0);
|
||||
if (item.publishedCount > 0 && !revisionNotifications) {
|
||||
const mangaId = item.libraryMangaId || (finalPath && path.basename(finalPath));
|
||||
if (mangaId) this.recordUpdateCheck({ ...item, libraryMangaId: mangaId }, { lastDownloadedAt: item.completed });
|
||||
}
|
||||
item.selectedCount = Number(result?.selectedCount ?? 0);
|
||||
this.state.history.push({ id: item.id, title: item.title, sourceTitle: item.sourceTitle, availability: item.availability, source, status: 'complete', time: item.completed, finalPath, publishedCount: item.publishedCount, selectedCount: item.selectedCount });
|
||||
await this.event('info', `Completed ${item.title}`, { id: item.id, title: item.title, source, finalPath, publishedCount: item.publishedCount });
|
||||
} catch (error) {
|
||||
if (error instanceof ArchiveControlError) {
|
||||
item.status = error.reason === 'quarantined' ? 'quarantined' : 'queued';
|
||||
delete item.stopReason;
|
||||
await this.event('info', `${item.title} ${error.reason}`, { id: item.id, title: item.title, source });
|
||||
return;
|
||||
}
|
||||
const failure = classifyFailure(error);
|
||||
item.failure = failure; item.error = failure.message;
|
||||
const incident = { id: crypto.randomUUID(), at: new Date().toISOString(), jobId: item.id, source, title: item.title, failure, state: 'open' };
|
||||
this.state.incidents.push(incident); this.state.incidents = this.state.incidents.slice(-200);
|
||||
if (failure.kind === 'unavailable') { item.status = 'unavailable'; incident.state = 'resolved'; incident.resolution = 'skipped-unavailable'; }
|
||||
else if (failure.retryable && (item.recoveryAttempts || 0) < 1) {
|
||||
item.recoveryAttempts = (item.recoveryAttempts || 0) + 1;
|
||||
item.status = 'retry_wait'; item.nextAttemptAt = new Date(Date.now() + this.retryDelayMs).toISOString();
|
||||
this.state.circuits[source] = { state: 'open', until: item.nextAttemptAt, jobId: item.id, reason: 'transient' };
|
||||
} else if (failure.scope === 'source') { item.status = 'error'; this.state.sourcePaused[source] = true; }
|
||||
else if (failure.scope === 'global') { item.status = 'error'; this.state.paused = true; }
|
||||
else {
|
||||
item.status = 'quarantined';
|
||||
const similar = this.state.incidents.filter(i => i.source === source && i.failure.kind === 'adapter' && Date.now() - Date.parse(i.at) < 3600000);
|
||||
if (new Set(similar.map(i => i.jobId)).size >= 2) this.state.sourcePaused[source] = true;
|
||||
}
|
||||
await this.event(failure.kind === 'unavailable' ? 'warning' : 'error', failure.message, { id: item.id, title: item.title, source, failure, nextAttemptAt: item.nextAttemptAt || null });
|
||||
}
|
||||
}
|
||||
|
||||
async workLoop() {
|
||||
while (!this.stopping) {
|
||||
this.runtimeHeartbeat = Date.now();
|
||||
if (!this.state.paused) {
|
||||
for (const item of this.state.queue) {
|
||||
if (item.status === 'retry_wait' && Date.parse(item.nextAttemptAt) <= Date.now()) { item.status = 'queued'; delete this.state.circuits[item.source]; }
|
||||
if (item.status !== 'queued') continue;
|
||||
const found = this.resolveAdapter(item.url);
|
||||
if (!found) {
|
||||
item.status = 'quarantined'; item.error = 'Adapter no longer available';
|
||||
await this.event('error', item.error, { id: item.id, title: item.title, source: item.source });
|
||||
continue;
|
||||
}
|
||||
const source = found.adapter.id;
|
||||
if (this.active.has(source) || this.state.sourcePaused[source]) continue;
|
||||
if (this.state.circuits[source]?.state === 'open') continue;
|
||||
if (item.libraryMangaId && [...this.active.values()].some(entry => {
|
||||
const activeItem = this.state.queue.find(candidate => candidate.id === entry.id);
|
||||
return activeItem?.libraryMangaId === item.libraryMangaId;
|
||||
})) continue;
|
||||
this.startItem(item, found);
|
||||
}
|
||||
}
|
||||
const prior = this.state.status;
|
||||
this.recomputeStatus();
|
||||
if (prior !== this.state.status) await this.persist();
|
||||
await sleep(500);
|
||||
}
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (!this.state) return;
|
||||
this.stopping = true;
|
||||
if (this.discoveryTimer) clearTimeout(this.discoveryTimer);
|
||||
this.discoveryTimer = null;
|
||||
this.recomputeStatus();
|
||||
await this.persist();
|
||||
const timeout = new Promise(resolve => { const timer = setTimeout(resolve, 30000); timer.unref(); });
|
||||
const workers = [...this.active.values()].map(entry => entry.promise).filter(Boolean);
|
||||
const drained = await Promise.race([Promise.allSettled([this.loopPromise || Promise.resolve(), ...workers]).then(() => true), timeout.then(() => false)]);
|
||||
await this.writeChain;
|
||||
// Keep the lock until the process exits if an in-flight worker did not drain.
|
||||
if (drained) await fs.rm(this.lockFile, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { ScraperService, DEFAULT_SETTINGS, validateSettings, settingsForSource };
|
||||
@@ -0,0 +1,71 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs').promises;
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { ScraperService } = require('../service');
|
||||
|
||||
const adapter = { id: 'alpha', label: 'Alpha',
|
||||
match(value) { return value.startsWith('https://alpha.test/') ? { id: '42', source: 'alpha', title: 'Test', url: value } : null; },
|
||||
async discover() { return { title: 'Test', chapters: [{ id: 'one', number: '1' }] }; },
|
||||
async run() { throw new Error('A no-change check must not download'); },
|
||||
};
|
||||
const resolve = value => { const info = adapter.match(value); return info ? { adapter, info } : null; };
|
||||
const waitFor = async predicate => {
|
||||
const deadline = Date.now() + 3000;
|
||||
while (!predicate()) {
|
||||
if (Date.now() > deadline) throw new Error('Timed out waiting for local discovery');
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 25));
|
||||
}
|
||||
};
|
||||
|
||||
test('finished-job cleanup preserves unfinished jobs, legacy check summaries, history and library files', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-scraper-dashboard-'));
|
||||
const dataRoot = path.join(root, 'data'), libraryRoot = path.join(root, 'library');
|
||||
await fs.mkdir(dataRoot); await fs.mkdir(path.join(libraryRoot, 'Test'), { recursive: true });
|
||||
const chapterFile = path.join(libraryRoot, 'Test', 'keep.jpg');
|
||||
await fs.writeFile(chapterFile, 'library-bytes');
|
||||
const queue = [
|
||||
{ id: 'done-a', kind: 'download', source: 'alpha', url: 'https://alpha.test/42', status: 'complete' },
|
||||
{ id: 'checked', kind: 'discovery', source: 'alpha', url: 'https://alpha.test/42', status: 'complete', libraryMangaId: 'Test', checked: '2026-10-06T12:00:00Z' },
|
||||
{ id: 'ready', kind: 'inspection', source: 'alpha', url: 'https://alpha.test/42', status: 'ready' },
|
||||
{ id: 'failed', kind: 'download', source: 'alpha', url: 'https://alpha.test/42', status: 'quarantined' },
|
||||
{ id: 'queued', kind: 'download', source: 'alpha', url: 'https://alpha.test/42', status: 'queued' },
|
||||
{ id: 'done-b', kind: 'download', source: 'beta', url: 'https://beta.test/42', status: 'complete' },
|
||||
];
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true, queue, history: [{ id: 'done-a', status: 'complete' }] }));
|
||||
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [adapter, { id: 'beta', label: 'Beta' }] }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
assert.deepEqual(service.state.queue.map(item => item.id), ['done-a', 'ready', 'failed', 'queued', 'done-b']);
|
||||
assert.equal((await service.publicState()).updateChecks[0].lastCheckedAt, '2026-10-06T12:00:00Z');
|
||||
assert.equal((await service.clearCompleted('alpha')).cleared, 1);
|
||||
assert.deepEqual(service.state.queue.map(item => item.id), ['ready', 'failed', 'queued', 'done-b']);
|
||||
assert.equal((await service.clearCompleted()).cleared, 1);
|
||||
assert.equal((await service.clearCompleted()).cleared, 0);
|
||||
assert.deepEqual(service.state.queue.map(item => item.id), ['ready', 'failed', 'queued']);
|
||||
assert.deepEqual(service.state.queue.map(item => item.order), [0, 1, 2]);
|
||||
assert.equal((await fs.readFile(chapterFile, 'utf8')), 'library-bytes');
|
||||
assert.equal(service.state.history[0].id, 'done-a');
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(dataRoot, 'queue.json'), 'utf8')).updateChecks['["Test","alpha"]'].lastCheckedAt, '2026-10-06T12:00:00Z');
|
||||
});
|
||||
|
||||
test('a nightly no-change check updates the latest timestamp and retires its queue row', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-scraper-check-summary-'));
|
||||
const dataRoot = path.join(root, 'data'), libraryRoot = path.join(root, 'library');
|
||||
const chapter = path.join(libraryRoot, 'Test', 'v01', 'c0001 [Alpha]');
|
||||
await fs.mkdir(dataRoot); await fs.mkdir(chapter, { recursive: true });
|
||||
await fs.writeFile(path.join(libraryRoot, 'Test', 'manga.json'), JSON.stringify({ title: 'Test', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }] }));
|
||||
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/one' } }));
|
||||
await fs.writeFile(path.join(chapter, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [adapter] }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
await service.enqueueDiscoveryChecks('schedule');
|
||||
await waitFor(() => service.state.updateChecks['["Test","alpha"]']?.lastCheckedAt);
|
||||
assert.equal(service.state.queue.length, 0);
|
||||
const summary = (await service.publicState()).updateChecks[0];
|
||||
assert.equal(summary.lastResult, 'no-updates');
|
||||
assert.equal(summary.lastNewContentAt, undefined);
|
||||
assert.equal(summary.lastDownloadedAt, undefined);
|
||||
assert.ok(service.state.history.some(item => item.status === 'checked'));
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { ScraperService, validateSettings } = require('../service');
|
||||
|
||||
function fixture(scanLibrary) {
|
||||
const service = new ScraperService({ scanLibrary, adapters: [] });
|
||||
service.state = { version: 4, paused: true, status: 'paused', queue: [], settings: validateSettings({}), currentIds: {} };
|
||||
service.recentEvents = async () => [];
|
||||
return service;
|
||||
}
|
||||
const inventory = title => ({ mangas: title ? [{ id: title, title, status: 'ongoing', autoUpdate: true, chapters: [], sources: [] }] : [] });
|
||||
|
||||
test('cold status returns controls immediately and simultaneous probes share one slow scan', { timeout: 1000 }, async () => {
|
||||
let finish, scans = 0;
|
||||
const service = fixture(() => { scans++; return new Promise(resolve => { finish = resolve; }); });
|
||||
const states = await Promise.all(Array.from({ length: 8 }, () => service.publicState()));
|
||||
assert.equal(scans, 1);
|
||||
for (const state of states) {
|
||||
assert.equal(state.paused, true);
|
||||
assert.equal(state.settings.sourceRates.mangadex.minMs, 1000);
|
||||
assert.deepEqual(state.library, []);
|
||||
assert.equal(state.libraryStatus.ready, false);
|
||||
assert.equal(state.libraryStatus.refreshing, true);
|
||||
}
|
||||
finish(inventory('Demo'));
|
||||
await service.inventoryRefresh;
|
||||
const ready = await service.publicState();
|
||||
assert.equal(ready.library[0].title, 'Demo');
|
||||
assert.equal(ready.libraryStatus.ready, true);
|
||||
assert.equal(ready.libraryStatus.stale, false);
|
||||
assert.equal(scans, 1);
|
||||
});
|
||||
|
||||
test('status keeps previous summary but authoritative callers await scans and reject invalidated results', { timeout: 1000 }, async () => {
|
||||
const completions = [];
|
||||
const service = fixture(() => new Promise(resolve => completions.push(resolve)));
|
||||
const first = service.libraryInventory(true);
|
||||
await Promise.resolve();
|
||||
completions[0](inventory('Old'));
|
||||
await first;
|
||||
service.invalidateInventory();
|
||||
const fresh = service.libraryInventory(true);
|
||||
await Promise.resolve();
|
||||
let finished = false;
|
||||
fresh.then(() => { finished = true; });
|
||||
let state = await service.publicState();
|
||||
assert.equal(state.library[0].title, 'Old');
|
||||
assert.equal(state.libraryStatus.stale, true);
|
||||
assert.equal(state.libraryStatus.refreshing, true);
|
||||
assert.equal(finished, false);
|
||||
// Publication during an in-flight scan requires another authoritative scan.
|
||||
service.invalidateInventory();
|
||||
const scan = service.inventoryRefresh;
|
||||
completions[1](inventory('Obsolete'));
|
||||
await scan;
|
||||
await Promise.resolve();
|
||||
assert.equal(completions.length, 3);
|
||||
assert.equal(finished, false);
|
||||
completions[2](inventory('Published'));
|
||||
assert.equal((await fresh).mangas[0].title, 'Published');
|
||||
state = await service.publicState();
|
||||
assert.equal(state.library[0].title, 'Published');
|
||||
assert.equal(state.libraryStatus.stale, false);
|
||||
});
|
||||
|
||||
test('background inventory failures are visible without breaking controls or launching a poll storm', { timeout: 1000 }, async () => {
|
||||
let scans = 0;
|
||||
const service = fixture(async () => { scans++; throw new Error('Scan failed https://test.example/a?token=private'); });
|
||||
await service.publicState();
|
||||
await assert.rejects(service.inventoryRefresh, /Scan failed/);
|
||||
for (let n = 0; n < 3; n++) {
|
||||
const state = await service.publicState();
|
||||
assert.equal(state.paused, true);
|
||||
assert.match(state.libraryStatus.error, /Scan failed/);
|
||||
assert.ok(!state.libraryStatus.error.includes('private'));
|
||||
}
|
||||
assert.equal(scans, 1);
|
||||
await assert.rejects(service.libraryInventory(true), /Scan failed/);
|
||||
assert.equal(scans, 2);
|
||||
});
|
||||
|
||||
test('dashboard polling reuses a recent snapshot while decisions still force a fresh scan', async () => {
|
||||
let scans = 0;
|
||||
const service = fixture(async () => { scans++; return inventory('Demo'); });
|
||||
await service.libraryInventory(true);
|
||||
service.inventoryCachedAt = Date.now() - 30000;
|
||||
for (let n = 0; n < 5; n++) assert.equal((await service.publicState()).library[0].title, 'Demo');
|
||||
assert.equal(scans, 1);
|
||||
await service.libraryInventory(true);
|
||||
assert.equal(scans, 2);
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { writeJsonAtomic } = require('../lib/json-file');
|
||||
|
||||
test('Windows rename fallback replaces JSON without deleting the old state first', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-json-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const file = path.join(root, 'queue.json');
|
||||
await fs.writeFile(file, '{"old":true}\n');
|
||||
let failedOnce = false;
|
||||
const io = { ...fs, async rename(from, to) {
|
||||
if (to === file && !failedOnce) {
|
||||
failedOnce = true;
|
||||
const error = new Error('destination busy'); error.code = 'EEXIST'; throw error;
|
||||
}
|
||||
return fs.rename(from, to);
|
||||
} };
|
||||
await writeJsonAtomic(file, { next: true }, io);
|
||||
assert.deepEqual(JSON.parse(await fs.readFile(file, 'utf8')), { next: true });
|
||||
assert.deepEqual(await fs.readdir(root), ['queue.json']);
|
||||
});
|
||||
|
||||
test('failed replacement restores the old JSON', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-json-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const file = path.join(root, 'queue.json');
|
||||
await fs.writeFile(file, '{"old":true}\n');
|
||||
const io = { ...fs, async rename(from, to) {
|
||||
if (to === file && from.endsWith('.tmp')) {
|
||||
const error = new Error('replacement blocked'); error.code = 'EEXIST'; throw error;
|
||||
}
|
||||
return fs.rename(from, to);
|
||||
} };
|
||||
await assert.rejects(writeJsonAtomic(file, { next: true }, io), /replacement blocked/);
|
||||
assert.deepEqual(JSON.parse(await fs.readFile(file, 'utf8')), { old: true });
|
||||
assert.deepEqual(await fs.readdir(root), ['queue.json']);
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises'), os = require('node:os'), path = require('node:path');
|
||||
const { specialKey, parseSpecialFolder } = require('../../chapter-identity');
|
||||
const { selectChapterReleases, availabilitySummary, discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
|
||||
const { selectByScope } = require('../lib/scope');
|
||||
const inventory = require('../lib/library-inventory');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const reader = require('../../library');
|
||||
const { ScraperService } = require('../service');
|
||||
const ids = ['37d6949a-c7ce-4855-b838-4fd572e87629', '27dd738c-eca2-488d-a791-53ab4b24313b'];
|
||||
const titleId = '06a37094-b3ba-4f25-bf2b-54c9751e0e91';
|
||||
const release = (id, chapter = null) => ({ id, attributes: { chapter, pages: 46, title: 'Pilot', externalUrl: null }, relationships: [] });
|
||||
const pause = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
async function until(predicate) {
|
||||
const end = Date.now() + 5000;
|
||||
while (!predicate()) { if (Date.now() > end) throw new Error('Fixture timed out'); await pause(20); }
|
||||
}
|
||||
|
||||
test('unnumbered releases retain source IDs, do not invent chapter zero, and remain distinct', async () => {
|
||||
const selected = selectChapterReleases(ids.map(id => release(id)), 'Pilot');
|
||||
assert.equal(selected.length, 2);
|
||||
assert.deepEqual(new Set(selected.map(r => r.number)), new Set(ids.map(specialKey)));
|
||||
assert.ok(selected.every(r => r.special && r.label === 'Pilot' && r.sourceNumber === null));
|
||||
assert.equal(selectByScope(selected, { mode: 'chapter', chapter: '0' }).length, 0);
|
||||
assert.equal(selectByScope(selected, { mode: 'after', chapter: '1' }).length, 0);
|
||||
assert.equal(selectByScope(selected, { mode: 'missing' }).length, 2);
|
||||
assert.equal(selectByScope(selected, { mode: 'ids', ids: [ids[0]] })[0].id, ids[0]);
|
||||
assert.throws(() => specialKey('../escape'));
|
||||
assert.equal(parseSpecialFolder(specialKey(ids[0]) + ' Pilot [MangaDex]').src, 'MangaDex');
|
||||
const chapter = await discoverMangaDexChapter(selected[0], { async get() { return { async json() { return { result: 'ok', baseUrl: 'https://uploads.mangadex.org', chapter: { hash: 'fixture', data: Array.from({ length: 46 }, (_, n) => `${n}.png`) } }; } }; } });
|
||||
assert.equal(chapter.label, 'Pilot'); assert.equal(chapter.sourceId, selected[0].id);
|
||||
});
|
||||
|
||||
test('availability distinguishes no English, external-only, and partial hosted catalogs', () => {
|
||||
assert.equal(availabilitySummary({ availableTranslatedLanguages: ['ar'] }, [], [], 0).reason, 'no-english');
|
||||
const external = availabilitySummary({}, [], [], 8);
|
||||
assert.equal(external.reason, 'external-only'); assert.match(external.message, /8 English external-link/);
|
||||
const chapters = selectChapterReleases([release('regular-fixture', '336')]);
|
||||
const summary = availabilitySummary({ lastChapter: '686' }, [release('regular-fixture', '336')], chapters);
|
||||
assert.equal(summary.coverage, 'partial'); assert.equal(summary.selectedChapters, 1);
|
||||
assert.equal(availabilitySummary({ lastChapter: '336' }, [], chapters).coverage, 'unknown', 'Highest number alone does not prove completeness');
|
||||
});
|
||||
|
||||
test('catalog uses separate external-only feed only when hosted selections are empty', async () => {
|
||||
const modes = [];
|
||||
const http = { async get(url) {
|
||||
const parsed = new URL(url), mode = parsed.searchParams.get('includeExternalUrl');
|
||||
if (!mode) return { async json() { return { data: { type: 'manga', attributes: { title: { en: 'CLAYMORE' }, lastChapter: '155' } } }; } };
|
||||
modes.push(mode);
|
||||
return { async json() { return { result: 'ok', data: [], total: mode === '1' ? 8 : 0 }; } };
|
||||
} };
|
||||
const catalog = await discoverMangaDexCatalog(`https://mangadex.org/title/${titleId}`, http);
|
||||
assert.deepEqual(modes, ['0', '1']); assert.equal(catalog.availability.reason, 'external-only');
|
||||
modes.length = 0;
|
||||
http.get = async url => {
|
||||
const mode = new URL(url).searchParams.get('includeExternalUrl');
|
||||
if (!mode) return { async json() { return { data: { type: 'manga', attributes: { title: { en: 'Bleach (Pre-Serialization)' } } } }; } };
|
||||
modes.push(mode); return { async json() { return { result: 'ok', data: ids.map(id => release(id)), total: 2 }; } };
|
||||
};
|
||||
const pilot = await discoverMangaDexCatalog(`https://mangadex.org/title/${titleId}`, http);
|
||||
assert.deepEqual(modes, ['0']); assert.equal(pilot.availability.specialReleases, 2); assert.equal(pilot.chapters[0].label, 'Pilot');
|
||||
});
|
||||
|
||||
test('special publication is atomic, readable, stable across rescans and duplicate-safe', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-special-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
// Real fixture PNG; no upstream image requests.
|
||||
const { Canvas } = require('../../tools/png');
|
||||
const page = path.join(root, '.fixture.png'); await fs.writeFile(page, new Canvas(10, 20, [1, 2, 3]).png());
|
||||
const manifest = { version: 1, series: { title: 'Pilot edition', direction: 'rtl', sources: [{ name: 'MangaDex', url: `https://mangadex.org/title/${titleId}` }] },
|
||||
chapters: selectChapterReleases(ids.map(id => release(id)), 'Pilot').map(ref => ({ ...ref, source: { name: 'MangaDex', url: ref.url }, pages: [{ path: page }] })) };
|
||||
const first = await publishManifest(manifest, { libraryRoot: root });
|
||||
assert.ok(first.results.every(result => result.status === 'published'));
|
||||
const scanned = (await reader.scanLibrary(root)).index.mangas[0];
|
||||
assert.equal(scanned.chapters.length, 2); assert.ok(scanned.chapters.every(c => c.label === 'Pilot' && c.num === null && !c.extra));
|
||||
const stableIds = scanned.chapters.map(c => c.id);
|
||||
assert.deepEqual((await publishManifest(manifest, { libraryRoot: root })).results.map(r => r.status), ['skipped', 'skipped']);
|
||||
assert.deepEqual((await reader.scanLibrary(root)).index.mangas[0].chapters.map(c => c.id), stableIds);
|
||||
const manga = (await inventory.scanLibrary(root)).mangas[0];
|
||||
assert.equal(inventory.shouldDownload(manga, manifest.chapters[0], 'mangadex'), false);
|
||||
assert.equal(inventory.isNewerThanLibrary(manga, manifest.chapters[0]), false);
|
||||
assert.equal(inventory.isNewerThanLibrary(manga, { number: '1' }), true);
|
||||
assert.equal(reader.parseKey('c0012.5 Existing [Host]').key, '12.5');
|
||||
assert.equal(reader.parseKey('v01').key, '1');
|
||||
const regular = { ...manifest, chapters: [{ number: '12.5', title: 'Existing chapter', source: { name: 'Host', url: 'https://host.test/12' }, pages: [{ path: page }] }] };
|
||||
await publishManifest(regular, { libraryRoot: root });
|
||||
assert.ok((await reader.scanLibrary(root)).index.mangas[0].chapterById.has('1/12.5'));
|
||||
});
|
||||
|
||||
test('legacy unavailable messages become unverified without retrying or downloading', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-unavailable-')), dataRoot = path.join(root, 'data');
|
||||
await fs.mkdir(dataRoot); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true,
|
||||
queue: [{ id: 'old-job', source: 'mangadex', url: `https://mangadex.org/title/${titleId}`, status: 'unavailable', error: 'No downloadable English MangaDex chapters found', failure: { kind: 'unavailable', scope: 'job' } }],
|
||||
settings: { autoUpdateEnabled: false } }));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(root, 'library'), dataRoot }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
assert.equal(service.state.queue[0].status, 'unavailable'); assert.equal(service.active.size, 0);
|
||||
assert.match(service.state.queue[0].error, /Inspect this source again/);
|
||||
assert.equal(service.state.queue[0].failure.kind, 'unavailable');
|
||||
});
|
||||
|
||||
test('inspection waits for confirmation, survives restart, and uses one normal source lane', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-inspect-')), dataRoot = path.join(root, 'data');
|
||||
await fs.mkdir(dataRoot); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true, queue: [], settings: { autoUpdateEnabled: false } }));
|
||||
let running = 0, maxRunning = 0, downloads = 0;
|
||||
const catalog = { title: 'Canonical Pilot', chapters: ids.map(id => ({ id, number: specialKey(id), special: true })), availability: { selectedChapters: 2, specialReleases: 2, coverage: 'unknown' } };
|
||||
const adapter = { id: 'fixture', label: 'Fixture', match: url => ({ source: 'fixture', id: url.split('/').pop(), url, title: 'Misleading slug' }),
|
||||
async discover() { maxRunning = Math.max(maxRunning, ++running); await pause(50); running--; return catalog; },
|
||||
async run(item, options) { downloads++; await options.onCatalog(catalog); return { publishedCount: 1, selectedCount: 1 }; } };
|
||||
const options = { libraryRoot: path.join(root, 'library'), dataRoot, adapters: [adapter], adapterFor: url => ({ adapter, info: adapter.match(url) }) };
|
||||
let service = await new ScraperService(options).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
const item = await service.enqueue({ url: 'https://fixture.test/one', inspect: true });
|
||||
const other = await service.enqueue({ url: 'https://fixture.test/two', inspect: true });
|
||||
await assert.rejects(service.control('confirm', item.id), /Inspect the source first/);
|
||||
await service.control('resume'); await until(() => item.status === 'ready' && other.status === 'ready');
|
||||
assert.equal(downloads, 0); assert.equal(maxRunning, 1); assert.equal(item.sourceTitle, 'Canonical Pilot');
|
||||
await service.close(); service = await new ScraperService(options).init();
|
||||
const restored = service.state.queue.find(q => q.id === item.id);
|
||||
assert.equal(restored.kind, 'inspection'); assert.equal(restored.status, 'ready');
|
||||
await service.control('confirm', restored.id); await until(() => restored.status === 'complete');
|
||||
await until(() => service.active.size === 0);
|
||||
assert.equal(downloads, 1); assert.equal(restored.selectedCount, 1);
|
||||
assert.equal(service.state.history.at(-1).availability.specialReleases, 2);
|
||||
service.state.sourcePaused.fixture = true;
|
||||
await assert.rejects(service.control('inspect', restored.id), /access block/);
|
||||
});
|
||||
|
||||
test('edition change after confirmation cannot silently download another edition', async () => {
|
||||
const service = Object.create(ScraperService.prototype);
|
||||
service.persist = async () => {};
|
||||
const item = { title: 'slug', confirmedSourceTitle: 'Pilot edition' };
|
||||
await assert.rejects(service.recordCatalog(item, { title: 'Main edition', chapters: [] }), /edition title changed/);
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs').promises;
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { detectVerticalStrip, resolveReadingDirection, fallbackDirection } = require('../../reading-direction');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const { scanLibrary } = require('../../library');
|
||||
|
||||
const strip = (w = 800) => ({ w, h: 12000 });
|
||||
|
||||
test('strip detection tolerates a landscape title card and rejects weak evidence', () => {
|
||||
const pages = [{ w: 1200, h: 800 }, strip(), strip(), strip()];
|
||||
assert.equal(detectVerticalStrip(pages).value, 'ttb');
|
||||
assert.equal(detectVerticalStrip(pages).evidence.tallPages, 3);
|
||||
assert.equal(detectVerticalStrip([strip()]), null);
|
||||
assert.equal(detectVerticalStrip([strip(), strip(), { w: 800, h: 1200 }, { w: 800, h: 1200 }]), null);
|
||||
assert.equal(detectVerticalStrip([strip(), strip(), strip(1400), { w: 1200, h: 800 }]), null);
|
||||
assert.equal(detectVerticalStrip([{ w: 800, h: 12000, r: 90 }, strip(), strip()]), null);
|
||||
assert.equal(detectVerticalStrip([{ w: 12000, h: 800, r: 90 }, strip(), strip()]).value, 'ttb');
|
||||
});
|
||||
|
||||
test('manual choices at chapter, book or manga level beat detection and fallback', () => {
|
||||
const pages = [strip(), strip(), strip()];
|
||||
const automatic = fallbackDirection();
|
||||
assert.equal(resolveReadingDirection([automatic], pages).value, 'ttb');
|
||||
assert.equal(resolveReadingDirection([automatic]).source, 'fallback');
|
||||
assert.equal(resolveReadingDirection([{ direction: 'ltr' }, automatic], pages).value, 'ltr');
|
||||
assert.equal(resolveReadingDirection([{}, { direction: 'rtl' }, automatic], pages).value, 'rtl');
|
||||
assert.equal(resolveReadingDirection([{ directionOverride: 'rtl', ...automatic }], pages).value, 'rtl');
|
||||
assert.equal(resolveReadingDirection([{ ...automatic, direction: 'ltr' }], pages).value, 'ltr');
|
||||
assert.equal(resolveReadingDirection([{ direction: 'rtl' }], pages).source, 'manual');
|
||||
assert.equal(resolveReadingDirection([], pages).value, 'ttb');
|
||||
});
|
||||
|
||||
function png(w, h) {
|
||||
const b = Buffer.alloc(24);
|
||||
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(b);
|
||||
b.writeUInt32BE(w, 16); b.writeUInt32BE(h, 20);
|
||||
return b;
|
||||
}
|
||||
|
||||
async function json(file) { return JSON.parse(await fs.readFile(file, 'utf8')); }
|
||||
|
||||
test('publisher detects new strips, preserves updates and manual metadata, and scanner agrees', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-direction-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const libraryRoot = path.join(temp, 'library');
|
||||
const sizes = [[1200, 800], [800, 12000], [800, 12500], [800, 14210]];
|
||||
const files = [];
|
||||
for (const [i, size] of sizes.entries()) {
|
||||
const file = path.join(temp, `page${i}.png`);
|
||||
await fs.writeFile(file, png(...size)); files.push(file);
|
||||
}
|
||||
const series = { title: 'Strip Work', ...fallbackDirection(),
|
||||
sources: [{ name: 'Test', url: 'https://example.test/series' }] };
|
||||
const chapter = number => ({ number, source: { name: 'Test', url: `https://example.test/${number}` },
|
||||
book: { number: 1, title: 'Season 1' }, pages: files.map(file => ({ path: file })) });
|
||||
const first = await publishManifest({ version: 1, series, chapters: [chapter('1')] }, { libraryRoot });
|
||||
const mangaDir = first.mangaDir, bookDir = path.join(mangaDir, 'v01 Season 1');
|
||||
const mangaFile = path.join(mangaDir, 'manga.json');
|
||||
assert.equal((await json(mangaFile)).direction, 'ttb');
|
||||
assert.equal((await json(mangaFile))['x-reading-direction'].source, 'detected');
|
||||
const scan = (await scanLibrary(libraryRoot)).index.mangas[0];
|
||||
assert.equal(scan.chapters[0].versions[0].direction, 'ttb');
|
||||
assert.equal(scan.chapters[0].versions[0].pages[0].w, 1200);
|
||||
|
||||
const same = await publishManifest({ version: 1, series, chapters: [chapter('2')] }, { libraryRoot });
|
||||
assert.equal(same.mangaDir, mangaDir);
|
||||
assert.equal((await json(mangaFile)).direction, 'ttb');
|
||||
|
||||
const bookFile = path.join(bookDir, 'book.json');
|
||||
await fs.writeFile(bookFile, JSON.stringify({ title: 'Season 1', direction: 'ltr', note: 'human' }));
|
||||
const chapterDir = path.join(bookDir, first.results[0].folder);
|
||||
const chapterFile = path.join(chapterDir, 'chapter.json');
|
||||
await fs.writeFile(chapterFile, JSON.stringify({ ...await json(chapterFile), directionOverride: 'rtl', note: 'human' }));
|
||||
await publishManifest({ version: 1, series, chapters: [chapter('1')] }, { libraryRoot, replace: true });
|
||||
assert.equal((await json(bookFile)).direction, 'ltr');
|
||||
assert.equal((await json(bookFile)).note, 'human');
|
||||
assert.equal((await json(chapterFile)).direction, 'rtl');
|
||||
assert.equal((await json(chapterFile)).directionOverride, 'rtl');
|
||||
assert.equal((await json(chapterFile)).note, 'human');
|
||||
const rescanned = (await scanLibrary(libraryRoot)).index.mangas[0];
|
||||
assert.equal(rescanned.chapters[0].versions[0].direction, 'rtl');
|
||||
assert.equal(rescanned.chapters[1].versions[0].direction, 'ltr');
|
||||
|
||||
await fs.writeFile(mangaFile, JSON.stringify({ ...await json(mangaFile), direction: 'rtl' }));
|
||||
await publishManifest({ version: 1, series, chapters: [chapter('3')] }, { libraryRoot });
|
||||
assert.equal((await json(mangaFile)).direction, 'rtl');
|
||||
assert.equal((await json(mangaFile))['x-reading-direction'].source, 'manual');
|
||||
assert.equal((await scanLibrary(libraryRoot)).index.mangas[0].chapters[2].versions[0].direction, 'ltr');
|
||||
});
|
||||
|
||||
test('legacy plain RTL stays protected across automatic imports', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-legacy-direction-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const libraryRoot = path.join(temp, 'library');
|
||||
const page = path.join(temp, 'tall.png'); await fs.writeFile(page, png(800, 12000));
|
||||
const series = { title: 'Legacy', ...fallbackDirection(), sources: [{ name: 'Test', url: 'https://example.test/legacy' }] };
|
||||
const chapter = { number: '1', source: { name: 'Test', url: 'https://example.test/legacy/1' },
|
||||
pages: [1, 2, 3].map(() => ({ path: page })) };
|
||||
const first = await publishManifest({ version: 1, series, chapters: [chapter] }, { libraryRoot });
|
||||
const file = path.join(first.mangaDir, 'manga.json');
|
||||
const meta = await json(file); delete meta['x-reading-direction']; meta.direction = 'rtl';
|
||||
await fs.writeFile(file, JSON.stringify(meta));
|
||||
await publishManifest({ version: 1, series, chapters: [{ ...chapter, number: '2', source: { name: 'Test', url: 'https://example.test/legacy/2' } }] }, { libraryRoot });
|
||||
assert.equal((await json(file)).direction, 'rtl');
|
||||
assert.equal((await json(file))['x-reading-direction'].source, 'manual');
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict'), fs = require('fs').promises, os = require('os'), path = require('path');
|
||||
const { ScraperService, validateSettings } = require('../service');
|
||||
const { classifyFailure, safeMessage } = require('../lib/failures');
|
||||
test('failure classification protects access challenges and distinguishes job failures', () => {
|
||||
for (const status of [401, 403, 429]) assert.equal(classifyFailure(Object.assign(new Error('Host response'), { status })).kind, 'access');
|
||||
assert.equal(classifyFailure(new Error('No downloadable English MangaDex chapters found')).scope, 'job');
|
||||
assert.equal(classifyFailure(Object.assign(new Error('Disk full'), { code: 'ENOSPC' })).scope, 'global');
|
||||
assert.equal(classifyFailure(new Error('HTTP 503')).retryable, true);
|
||||
assert.equal(safeMessage('HTTP 403 https://test.example/a?token=secret'), 'HTTP 403 https://test.example/a');
|
||||
});
|
||||
async function fixture(t, errors, options = {}) {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-recovery-'));
|
||||
const dataRoot = path.join(dir, 'data'); await fs.mkdir(dataRoot);
|
||||
const counts = {};
|
||||
const adapter = { id: 'test', label: 'Test', run: async item => {
|
||||
counts[item.id] = (counts[item.id] || 0) + 1;
|
||||
if (errors[item.id]) throw errors[item.id]; return { publishedCount: 0 };
|
||||
} };
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 3, paused: true, settings: validateSettings({ autoUpdateEnabled: false }),
|
||||
queue: Object.keys(errors).concat('healthy').map(id => ({ id, seriesId: id, source: 'test', title: id, url: 'https://test.example/' + id, status: 'queued' })), history: [] }));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(dir, 'library'), dataRoot, adapterFor: url => ({ adapter, info: { id: url.split('/').pop() } }), adapters: [adapter], ...options }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(dir, { recursive: true, force: true }); });
|
||||
await service.control('resume');
|
||||
return { service, counts };
|
||||
}
|
||||
async function wait(predicate) {
|
||||
const end = Date.now() + 6000;
|
||||
while (!predicate()) { if (Date.now() > end) throw new Error('Timeout'); await new Promise(r => setTimeout(r, 25)); }
|
||||
}
|
||||
test('unavailable titles and one parser failure do not stop healthy jobs', async t => {
|
||||
const { service } = await fixture(t, { absent: new Error('No downloadable English MangaDex chapters found'), parser: new Error('Catalog markup changed') });
|
||||
await wait(() => service.state.queue.find(i => i.id === 'healthy').status === 'complete');
|
||||
assert.equal(service.state.queue.find(i => i.id === 'absent').status, 'unavailable');
|
||||
assert.equal(service.state.queue.find(i => i.id === 'parser').status, 'quarantined');
|
||||
assert.equal(service.state.sourcePaused.test, undefined);
|
||||
});
|
||||
test('transient recovery is bounded and access errors are never retried automatically', async t => {
|
||||
const { service, counts } = await fixture(t, { broken: new Error('HTTP 503') }, { retryDelayMs: 10 });
|
||||
await wait(() => service.state.sourcePaused.test);
|
||||
assert.equal(counts.broken, 2); assert.equal(service.state.queue[0].status, 'error');
|
||||
});
|
||||
test('protection failures pause only the affected lane and keep checkpoints', async t => {
|
||||
const { service, counts } = await fixture(t, { blocked: new Error('HTTP 403') });
|
||||
await wait(() => service.state.sourcePaused.test);
|
||||
assert.equal(counts.blocked, 1); assert.equal(counts.healthy, undefined);
|
||||
assert.equal(service.state.incidents[0].failure.kind, 'access');
|
||||
});
|
||||
@@ -0,0 +1,556 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { cleanPart, chapterKey, chapterFolder, assertInside } = require('../lib/names');
|
||||
const { imageExtension } = require('../lib/images');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const { mergeImportedMetadata } = require('../lib/metadata-merge');
|
||||
const { metadataFromMedia } = require('../metadata/anilist');
|
||||
const { normalizeScope, selectByScope, scopeLabel } = require('../lib/scope');
|
||||
const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, setAutoUpdate } = require('../lib/library-inventory');
|
||||
const { inspectLegacyWork } = require('../lib/legacy-publication');
|
||||
const { discoverWeebCatalog, parseChapters, parsePageUrls } = require('../adapters/weebcentral');
|
||||
const { chapterNumber, selectChapterReleases, pageUrls } = require('../adapters/mangadex');
|
||||
const { adapterFor } = require('../adapters');
|
||||
const { ScraperService, validateSettings, settingsForSource } = require('../service');
|
||||
|
||||
test('MangaCircus chapter keys are normalized', () => {
|
||||
assert.equal(chapterKey('12'), 'c0012');
|
||||
assert.equal(chapterKey('12.50'), 'c0012.5');
|
||||
assert.equal(chapterKey('x2'), 'x02');
|
||||
assert.throws(() => chapterKey('special'));
|
||||
});
|
||||
|
||||
test('folder names are portable and versioned', () => {
|
||||
assert.equal(cleanPart('Bad:<Title>. '), 'BadTitle');
|
||||
assert.equal(cleanPart('CON'), 'CON-manga');
|
||||
assert.equal(chapterFolder({ number: '7', title: 'A [Test]', source: { name: 'Host' } }), 'c0007 A Test [Host]');
|
||||
assert.throws(() => assertInside('C:/library', 'C:/outside'));
|
||||
});
|
||||
|
||||
test('image signatures are recognized', () => {
|
||||
assert.equal(imageExtension(Buffer.from([0xff, 0xd8, 0xff, 0x00])), 'jpg');
|
||||
assert.equal(imageExtension(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])), 'png');
|
||||
assert.equal(imageExtension(Buffer.from('not an image')), null);
|
||||
});
|
||||
|
||||
test('per-source rate settings default fast and update independently', () => {
|
||||
const defaults = validateSettings({});
|
||||
const fast = { minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000 };
|
||||
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast);
|
||||
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), fast);
|
||||
|
||||
const changed = validateSettings({ sourceRates: { mangadex: {
|
||||
minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000,
|
||||
} } }, defaults);
|
||||
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), {
|
||||
minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000,
|
||||
});
|
||||
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast);
|
||||
assert.throws(() => validateSettings({ sourceRates: { mangadex: { minMs: 5000, maxMs: 1000 } } }, defaults));
|
||||
});
|
||||
|
||||
test('AniList metadata keeps manga format and original story credits', () => {
|
||||
const metadata = metadataFromMedia({
|
||||
id: 150193, idMal: null, format: 'MANGA',
|
||||
title: { english: 'Heavy Knight', romaji: 'Juukishi', native: '重騎士' }, synonyms: [],
|
||||
staff: { edges: [
|
||||
{ role: 'Original Story', node: { name: { full: 'Nekoko' } } },
|
||||
{ role: 'Story & Art', node: { name: { full: 'Lee Brocco' } } },
|
||||
{ role: 'Character Design', node: { name: { full: 'Jaian' } } },
|
||||
] },
|
||||
status: 'RELEASING', startDate: { year: 2022 }, genres: ['Fantasy'], description: '',
|
||||
siteUrl: 'https://anilist.co/manga/150193', coverImage: { extraLarge: 'https://s4.anilist.co/cover.jpg' },
|
||||
});
|
||||
assert.deepEqual(metadata.authors, ['Nekoko', 'Lee Brocco']);
|
||||
assert.equal(metadata['x-anilist'].format, 'MANGA');
|
||||
});
|
||||
|
||||
test('metadata refresh preserves legacy and manual fields while updating untouched imports', () => {
|
||||
const first = mergeImportedMetadata({ title: 'My preferred title', status: 'ongoing' }, {
|
||||
title: 'Provider title', status: 'completed', tags: ['Fantasy'], 'x-anilist': { id: 42 },
|
||||
}, 'Fallback');
|
||||
assert.equal(first.title, 'My preferred title');
|
||||
assert.equal(first.status, 'ongoing');
|
||||
assert.deepEqual(first.tags, ['Fantasy']);
|
||||
assert.equal(first['x-metadata'].imported.title, 'Provider title');
|
||||
const second = mergeImportedMetadata({ ...first, tags: ['Fantasy', 'Favorite'] }, {
|
||||
title: 'New provider title', status: 'ongoing', tags: ['Action'], 'x-anilist': { id: 42 },
|
||||
}, 'Fallback');
|
||||
assert.equal(second.title, 'My preferred title');
|
||||
assert.equal(second.status, 'ongoing');
|
||||
assert.deepEqual(second.tags, ['Fantasy', 'Favorite']);
|
||||
const third = mergeImportedMetadata(second, { description: 'First', 'x-anilist': { id: 42 } }, 'Fallback');
|
||||
assert.equal(mergeImportedMetadata(third, { description: 'Revised', 'x-anilist': { id: 42 } }, 'Fallback').description, 'Revised');
|
||||
assert.throws(() => mergeImportedMetadata(first, { 'x-anilist': { id: 99 } }, 'Fallback'), /identity conflict/);
|
||||
});
|
||||
|
||||
test('publisher rejects a conflicting work ID before fetching any pages', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-identity-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const manga = path.join(root, 'Existing');
|
||||
await fs.mkdir(manga);
|
||||
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Existing',
|
||||
sources: [{ name: 'Host', url: 'https://example.test/series' }], 'x-anilist': { id: 42 } }));
|
||||
let requested = 0;
|
||||
await assert.rejects(publishManifest({ version: 1,
|
||||
series: { title: 'Existing', direction: 'rtl', metadata: { 'x-anilist': { id: 99 } },
|
||||
sources: [{ name: 'Host', url: 'https://example.test/series' }] },
|
||||
chapters: [{ number: '1', source: { name: 'Host', url: 'https://example.test/chapter' },
|
||||
pages: [{ url: 'https://example.test/page.png' }] }],
|
||||
}, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }), /identity conflict/);
|
||||
assert.equal(requested, 0);
|
||||
assert.equal((await fs.readdir(manga)).includes('v01'), false);
|
||||
});
|
||||
|
||||
test('queue scopes select exact catalog slices', () => {
|
||||
const chapters = [
|
||||
{ id: 'a', number: '1', volume: '1' }, { id: 'b', number: '2', volume: '1' },
|
||||
{ id: 'c', number: '3', volume: '2' }, { id: 'd', number: '4', volume: '2' },
|
||||
];
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'latest' }).map(x => x.id), ['d']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'last', count: 2 }).map(x => x.id), ['c', 'd']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'first', count: 2 }).map(x => x.id), ['a', 'b']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'chapter', chapter: '003.0' }).map(x => x.id), ['c']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'after', chapter: '2' }).map(x => x.id), ['c', 'd']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'volume', volume: '2' }).map(x => x.id), ['c', 'd']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'ids', ids: ['b', 'd'] }).map(x => x.id), ['b', 'd']);
|
||||
assert.equal(scopeLabel(normalizeScope({ mode: 'last', count: 5 })), 'last 5 chapters');
|
||||
});
|
||||
|
||||
test('library inventory distinguishes missing chapters from alternate source versions', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-inventory-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const manga = path.join(temp, 'Library Manga'), book = path.join(manga, 'v01');
|
||||
await fs.mkdir(path.join(book, 'c0001 [WeebCentral]'), { recursive: true });
|
||||
await fs.mkdir(path.join(book, 'c0002 [MangaDex]'), { recursive: true });
|
||||
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({
|
||||
title: 'Library Manga', status: 'ongoing',
|
||||
sources: [{ name: 'WeebCentral', url: 'https://weebcentral.com/series/SERIES/Test' }],
|
||||
'x-scraper': { ids: { weebcentral: 'SERIES', anilist: 42 } },
|
||||
}));
|
||||
await fs.writeFile(path.join(book, 'c0001 [WeebCentral]', 'chapter.json'), JSON.stringify({ source: { name: 'WeebCentral', url: 'https://weebcentral.com/chapters/A' } }));
|
||||
await fs.writeFile(path.join(book, 'c0002 [MangaDex]', 'chapter.json'), JSON.stringify({ source: { name: 'MangaDex · Group', url: 'https://mangadex.org/chapter/B' } }));
|
||||
const inventory = await scanLibrary(temp);
|
||||
const found = findManga(inventory, { source: 'weebcentral', seriesId: 'SERIES' });
|
||||
assert.equal(found.id, 'Library Manga');
|
||||
assert.equal(findManga(inventory, { source: 'mangadex', seriesId: 'other', anilist: 42 }).id, 'Library Manga');
|
||||
assert.equal(found.autoUpdate, true);
|
||||
assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'skip-any'), false);
|
||||
assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'add-source'), true);
|
||||
assert.equal(shouldDownload(found, { number: '1' }, 'weebcentral', 'add-source'), false);
|
||||
assert.equal(shouldDownload(found, { number: '3' }, 'weebcentral', 'skip-any'), true);
|
||||
assert.equal(isNewerThanLibrary(found, { number: '1' }), false);
|
||||
assert.equal(isNewerThanLibrary(found, { number: '3' }), true);
|
||||
await setAutoUpdate(temp, 'Library Manga', false);
|
||||
assert.equal((await scanLibrary(temp)).mangas[0].autoUpdate, false);
|
||||
});
|
||||
|
||||
test('a shared work ID cannot merge distinct source listings or ambiguous editions', () => {
|
||||
const original = { id: 'Main', ids: { weebcentral: 'MAIN', anilist: 42 }, sources: [] };
|
||||
assert.equal(findManga({ mangas: [original] }, { source: 'weebcentral', seriesId: 'PILOT', anilist: 42 }), null);
|
||||
assert.equal(findManga({ mangas: [original] }, { source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), original);
|
||||
assert.equal(findManga({ mangas: [original, { id: 'Pilot', ids: { anilist: 42 }, sources: [] }] },
|
||||
{ source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), null);
|
||||
});
|
||||
|
||||
test('recent events read the bounded log tail without losing the newest incident', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-events-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const service = new ScraperService({ dataRoot: root });
|
||||
await fs.writeFile(service.logFile, `${'x'.repeat(600 * 1024)}\n${JSON.stringify({ level: 'error', message: 'recent' })}\n`);
|
||||
assert.deepEqual(await service.recentEvents(1), [{ level: 'error', message: 'recent' }]);
|
||||
assert.equal((await service.recentEvents(10)).at(-1).message, 'recent');
|
||||
});
|
||||
|
||||
test('paused legacy work is validated, published, and converted to incremental mode', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-legacy-publish-'));
|
||||
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
||||
const workRoot = path.join(libraryRoot, '.mcscraper-work-SERIES42');
|
||||
const stagedManga = path.join(workRoot, 'Legacy Manga'), book = path.join(stagedManga, 'v01');
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.mkdir(book, { recursive: true });
|
||||
await fs.writeFile(path.join(stagedManga, 'manga.json'), JSON.stringify({
|
||||
title: 'Legacy Manga', status: 'ongoing',
|
||||
sources: [{ name: 'Alpha', url: 'https://alpha.test/title/SERIES42' }],
|
||||
'x-scraper': { ids: { alpha: 'SERIES42' } },
|
||||
}));
|
||||
for (const number of [1, 2]) {
|
||||
const chapter = path.join(book, `c000${number} [Alpha]`);
|
||||
await fs.mkdir(chapter, { recursive: true });
|
||||
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: `https://alpha.test/chapter/${number}` } }));
|
||||
await fs.writeFile(path.join(chapter, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
}
|
||||
const incomplete = path.join(book, '.staging-mcscraper-abc123');
|
||||
await fs.mkdir(incomplete);
|
||||
await fs.writeFile(path.join(incomplete, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
await fs.writeFile(path.join(workRoot, 'job.json'), JSON.stringify({
|
||||
version: 2, status: 'paused', completed: ['one', 'two'], total: 4,
|
||||
}));
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 3, paused: true, status: 'paused', settings: validateSettings({}), history: [], publishedRevision: 0,
|
||||
queue: [{ id: 'old-id', seriesId: 'SERIES42', source: 'alpha', title: 'Legacy Manga', url: 'https://alpha.test/title/SERIES42', status: 'queued', order: 0 }],
|
||||
}));
|
||||
const alpha = {
|
||||
id: 'alpha', label: 'Alpha',
|
||||
match(value) { return String(value).includes('alpha.test/title/') ? { source: 'alpha', id: 'SERIES42', title: 'Legacy Manga', url: value } : null; },
|
||||
async discover() { return { chapters: [] }; }, async run() { return null; },
|
||||
};
|
||||
const service = await new ScraperService({
|
||||
libraryRoot, dataRoot, adapters: [alpha],
|
||||
adapterFor: value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; },
|
||||
}).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
||||
|
||||
const inspection = await inspectLegacyWork({ libraryRoot, seriesId: 'SERIES42' });
|
||||
assert.equal(inspection.chapterCount, 2);
|
||||
assert.equal(inspection.pageCount, 2);
|
||||
assert.equal(inspection.incompleteStages.length, 1);
|
||||
const result = await service.publishLegacyReady();
|
||||
assert.equal(result.published.length, 1);
|
||||
assert.equal(result.published[0].chaptersPublished, 2);
|
||||
assert.equal(result.published[0].incompleteStagesQuarantined, 1);
|
||||
assert.equal(await fs.stat(path.join(libraryRoot, 'Legacy Manga', 'v01', 'c0002 [Alpha]')).then(stat => stat.isDirectory()), true);
|
||||
await assert.rejects(fs.stat(stagedManga));
|
||||
const migrated = JSON.parse(await fs.readFile(path.join(workRoot, 'job.json'), 'utf8'));
|
||||
assert.equal(migrated.publicationMode, 'incremental');
|
||||
assert.equal(migrated.migratedToIncremental.chaptersPublished, 2);
|
||||
assert.equal(migrated.migratedToIncremental.incompleteStagesQuarantined, 1);
|
||||
assert.equal(await fs.stat(path.join(workRoot, '.legacy-incomplete', 'v01__.staging-mcscraper-abc123')).then(stat => stat.isDirectory()), true);
|
||||
assert.equal(service.state.queue[0].libraryMangaId, 'Legacy Manga');
|
||||
assert.equal(service.state.publishedRevision, 1);
|
||||
});
|
||||
|
||||
test('WeebCentral fragments preserve chapter and page order', () => {
|
||||
const chapters = parseChapters('<a href="/chapters/P1">Prologue 1</a><a href="/chapters/ABC">Chapter 12</a><a href="/chapters/DEF">Chapter 12.5</a><a href="/chapters/S2">S2 - Chapter 1</a><a href="/chapters/R442"><span>Rating 442</span><span>Last Read</span></a><a href="/chapters/E128"><span>Episode 128</span><span>Last Read</span></a><a href="/chapters/E78"><span>Episode: 78</span><span>Last Read</span></a><a href="/chapters/N182"><span>Night 182</span><span>Last Read</span></a><a href="/chapters/L65">Log. 65</a><a href="/chapters/A606">Act 60.6</a>', 'https://weebcentral.com/s');
|
||||
assert.deepEqual(chapters.map(item => item.number), ['0.01', '12', '12.5', '1', '442', '128', '78', '182', '65', '60.6']);
|
||||
assert.deepEqual(chapters[3].book, { number: 2, title: 'Season 2' });
|
||||
const pages = parsePageUrls('<img src="/001.png"><img data-src="https://img.example/002.webp">', 'https://weebcentral.com/c');
|
||||
assert.deepEqual(pages, ['https://weebcentral.com/001.png', 'https://img.example/002.webp']);
|
||||
});
|
||||
|
||||
test('WeebCentral reports unrecognized chapter labels separately from an empty catalog', async () => {
|
||||
const series = '<h1>Example</h1><div hx-get="/series/X/full-chapter-list"></div>';
|
||||
const httpFor = list => ({ get: async url => ({ text: async () => url.endsWith('full-chapter-list') ? list : series }) });
|
||||
const unknown = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<a href="/chapters/ABC">Part One</a>'));
|
||||
assert.equal(unknown.chapterLinks, 1);
|
||||
assert.equal(unknown.chapters.length, 0);
|
||||
assert.equal(unknown.availability.reason, 'unrecognized-labels');
|
||||
assert.match(unknown.availability.message, /1 chapter links/);
|
||||
const empty = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<div>No chapters</div>'));
|
||||
assert.equal(empty.chapterLinks, 0);
|
||||
assert.equal(empty.availability, undefined);
|
||||
});
|
||||
|
||||
test('completed download reinspection preserves complete state when all scoped chapters are present', async () => {
|
||||
const service = new ScraperService({ libraryRoot: 'unused', dataRoot: 'unused' });
|
||||
const item = { id: 'done', title: 'Done', source: 'weebcentral', url: 'https://weebcentral.com/series/EXAMPLE/Done',
|
||||
seriesId: 'EXAMPLE', kind: 'inspection', status: 'queued', completed: '2026-10-01T00:00:00Z',
|
||||
publishedCount: 2, selectedCount: 2, scope: { mode: 'missing' }, inspectionFromComplete: true };
|
||||
service.state = { paused: false, settings: validateSettings({}), queue: [item], incidents: [], sourcePaused: {}, currentIds: {} };
|
||||
service.recordCatalog = async (entry, catalog) => { entry.sourceTitle = catalog.title; entry.catalogTotal = catalog.chapters.length; };
|
||||
service.libraryInventory = async () => ({ mangas: [{ id: 'Done', ids: { weebcentral: 'EXAMPLE' }, sources: [],
|
||||
chapters: [{ book: '1', number: '1' }, { book: '1', number: '2' }] }] });
|
||||
service.event = async () => {};
|
||||
const adapter = { discover: async () => ({ title: 'Done', chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }] }) };
|
||||
await service.runDiscovery(item, { adapter }, 'weebcentral');
|
||||
assert.equal(item.status, 'complete');
|
||||
assert.equal(item.kind, 'download');
|
||||
assert.equal(item.completed, '2026-10-01T00:00:00Z');
|
||||
assert.equal(item.publishedCount, 2);
|
||||
assert.equal(item.inspectionFromComplete, undefined);
|
||||
});
|
||||
|
||||
test('publisher keeps season chapter numbers in separate books', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-books-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const page = path.join(temp, 'page.png');
|
||||
await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
const base = { title: '', source: { name: 'Host', url: 'https://example.test/chapter' }, pages: [{ path: page }] };
|
||||
await publishManifest({
|
||||
version: 1,
|
||||
series: { title: 'Season Test', direction: 'ttb', sources: [{ name: 'Host', url: 'https://example.test/series' }] },
|
||||
chapters: [
|
||||
{ ...base, number: '1', book: { number: 1, title: 'Season 1' } },
|
||||
{ ...base, number: '1', book: { number: 2, title: 'Season 2' } },
|
||||
],
|
||||
}, { libraryRoot: path.join(temp, 'library') });
|
||||
assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v01 Season 1', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 1');
|
||||
assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v02 Season 2', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 2');
|
||||
});
|
||||
|
||||
test('publisher stages and atomically exposes a complete new manga', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-test-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const page = path.join(temp, 'page.png');
|
||||
await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
const result = await publishManifest({
|
||||
version: 1,
|
||||
series: { title: 'Test Manga', direction: 'rtl', metadata: { status: 'ongoing' },
|
||||
sources: [{ name: 'Test', url: 'https://example.test/manga' }] },
|
||||
chapters: [{ number: '1', title: 'Start', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }],
|
||||
}, { libraryRoot: path.join(temp, 'library') });
|
||||
assert.equal(result.results[0].status, 'published');
|
||||
const chapter = path.join(temp, 'library', 'Test Manga', 'v01', 'c0001 Start [Test]');
|
||||
assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', 'chapter.json']);
|
||||
const meta = JSON.parse(await fs.readFile(path.join(temp, 'library', 'Test Manga', 'manga.json'), 'utf8'));
|
||||
assert.equal(meta['x-scraper'].autoUpdate, undefined, 'ongoing is a derived default, not a frozen override');
|
||||
assert.equal((await scanLibrary(path.join(temp, 'library'))).mangas[0].autoUpdate, true);
|
||||
assert.equal((await fs.readdir(path.join(temp, 'library'))).some(name => name.startsWith('.staging-')), false);
|
||||
});
|
||||
|
||||
test('publisher installs a validated AniList series cover', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-cover-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const page = path.join(temp, 'page.png');
|
||||
const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]);
|
||||
await fs.writeFile(page, png);
|
||||
let requests = 0;
|
||||
const httpClient = {
|
||||
async get(url) {
|
||||
requests++;
|
||||
assert.equal(url, 'https://s4.anilist.co/example.png');
|
||||
return { arrayBuffer: async () => png };
|
||||
},
|
||||
};
|
||||
const series = {
|
||||
title: 'Covered Manga', direction: 'rtl',
|
||||
sources: [{ name: 'Test', url: 'https://example.test/covered' }],
|
||||
coverUrl: 'https://s4.anilist.co/example.png',
|
||||
};
|
||||
const result = await publishManifest({
|
||||
version: 1, series,
|
||||
chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }],
|
||||
}, { libraryRoot: path.join(temp, 'library'), httpClient });
|
||||
assert.equal(result.cover.status, 'installed');
|
||||
assert.deepEqual(await fs.readFile(path.join(temp, 'library', 'Covered Manga', 'cover.png')), png);
|
||||
assert.equal(requests, 1);
|
||||
});
|
||||
|
||||
test('publisher retains validated partial pages and resumes without duplicate requests', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-resume-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]);
|
||||
const manifest = {
|
||||
version: 1, series: { title: 'Resumable', direction: 'rtl', sources: [{ name: 'Test', url: 'https://example.test/series' }] },
|
||||
chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' },
|
||||
pages: [1, 2].map(n => ({ url: `https://example.test/${n}.png` })) }],
|
||||
};
|
||||
const requests = [];
|
||||
const httpClient = { async get(url) { requests.push(url); return { arrayBuffer: async () => png }; } };
|
||||
const libraryRoot = path.join(temp, 'library');
|
||||
await assert.rejects(publishManifest(manifest, { libraryRoot, httpClient,
|
||||
onPage() { throw new Error('cooperative pause'); } }), /cooperative pause/);
|
||||
await assert.rejects(fs.stat(path.join(libraryRoot, 'Resumable')));
|
||||
const hidden = (await fs.readdir(libraryRoot))[0];
|
||||
assert.match(hidden, /^\.staging-mcscraper-resume-/);
|
||||
const { cleanupOwnTemps } = require('../lib/publisher');
|
||||
await cleanupOwnTemps(libraryRoot, -1);
|
||||
assert.ok(await fs.stat(path.join(libraryRoot, hidden)));
|
||||
await publishManifest(manifest, { libraryRoot, httpClient });
|
||||
assert.deepEqual(requests, ['https://example.test/1.png', 'https://example.test/2.png']);
|
||||
const chapter = path.join(libraryRoot, 'Resumable', 'v01', 'c0001 [Test]');
|
||||
assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', '0002.png', 'chapter.json']);
|
||||
});
|
||||
|
||||
test('adapter registry recognizes supported series URLs', () => {
|
||||
const found = adapterFor('https://weebcentral.com/series/01ABCDEF/My-Series');
|
||||
assert.equal(found.adapter.id, 'weebcentral');
|
||||
assert.equal(found.info.id, '01ABCDEF');
|
||||
assert.equal(found.info.title, 'My Series');
|
||||
const dex = adapterFor('https://mangadex.org/title/5bdb093f-9707-4a30-b02e-c8b66554ecf0/biomega');
|
||||
assert.equal(dex.adapter.id, 'mangadex');
|
||||
assert.equal(dex.info.id, '5bdb093f-9707-4a30-b02e-c8b66554ecf0');
|
||||
assert.equal(adapterFor('https://example.test/series/1'), null);
|
||||
});
|
||||
|
||||
test('MangaDex releases are normalized, deduplicated, credited, and host-validated', () => {
|
||||
assert.equal(chapterNumber('0012.50'), '12.5');
|
||||
assert.equal(chapterNumber('Special'), null);
|
||||
const release = (id, chapter, group, publishAt) => ({
|
||||
id, attributes: { chapter, title: '', pages: 2, publishAt, externalUrl: null },
|
||||
relationships: group ? [{ id: `group-${id}`, type: 'scanlation_group', attributes: { name: group } }] : [],
|
||||
});
|
||||
const selected = selectChapterReleases([
|
||||
release('fan', '1', 'Fan Group', '2020-01-01T00:00:00Z'),
|
||||
release('official', '1.0', 'Publisher Official', '2021-01-01T00:00:00Z'),
|
||||
release('two', '2', 'Fan Group', '2020-01-02T00:00:00Z'),
|
||||
]);
|
||||
assert.deepEqual(selected.map(item => item.id), ['official', 'two']);
|
||||
assert.equal(selected[0].group.name, 'Publisher Official');
|
||||
const pages = pageUrls({
|
||||
baseUrl: 'https://node.mangadex.network',
|
||||
chapter: { hash: 'abc', data: ['1.jpg', '2.png'] },
|
||||
}, 'https://mangadex.org/chapter/test', 2);
|
||||
assert.deepEqual(pages.map(item => item.url), [
|
||||
'https://node.mangadex.network/data/abc/1.jpg',
|
||||
'https://node.mangadex.network/data/abc/2.png',
|
||||
]);
|
||||
assert.throws(() => pageUrls({ baseUrl: 'https://evil.example', chapter: { hash: 'abc', data: ['1.jpg'] } }, 'https://mangadex.org/chapter/test', 1));
|
||||
});
|
||||
|
||||
test('daemon queue state is durable while paused', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-service-'));
|
||||
const dataRoot = path.join(temp, 'data');
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 1, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
||||
}));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
||||
const item = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' });
|
||||
assert.equal(item.status, 'queued');
|
||||
const scoped = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series', scope: { mode: 'latest' } });
|
||||
assert.notEqual(item.id, scoped.id);
|
||||
assert.equal(item.seriesId, scoped.seriesId);
|
||||
assert.equal((await service.publicState()).queue.length, 2);
|
||||
await service.updateSettings({ minMs: 1500, maxMs: 4500 });
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(dataRoot, 'queue.json'), 'utf8')).settings.minMs, 1500);
|
||||
await service.remove(item.id);
|
||||
await service.remove(scoped.id);
|
||||
assert.equal((await service.publicState()).queue.length, 0);
|
||||
});
|
||||
|
||||
test('retry recovers stale downloading entries before work resumes', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-retry-'));
|
||||
const dataRoot = path.join(temp, 'data');
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 1, paused: true, status: 'error', currentId: 'FAILED', settings: validateSettings({}),
|
||||
queue: [
|
||||
{ id: 'STALE', title: 'Stale', url: 'https://weebcentral.com/series/STALE/Stale', status: 'downloading', order: 0 },
|
||||
{ id: 'FAILED', title: 'Failed', url: 'https://weebcentral.com/series/FAILED/Failed', status: 'error', order: 1 },
|
||||
], history: [], publishedRevision: 0,
|
||||
}));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
||||
await service.control('retry', 'FAILED');
|
||||
assert.deepEqual(service.state.queue.map(item => item.status), ['queued', 'queued']);
|
||||
assert.equal(service.state.currentId, null);
|
||||
assert.equal(service.state.paused, false);
|
||||
});
|
||||
|
||||
test('nightly discovery queues one preferred supported source for ongoing manga', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-'));
|
||||
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
||||
const manga = path.join(libraryRoot, 'Ongoing Manga');
|
||||
await fs.mkdir(manga, { recursive: true });
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({
|
||||
title: 'Ongoing Manga', status: 'ongoing',
|
||||
sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }],
|
||||
'x-scraper': { ids: { alpha: '42' } },
|
||||
}));
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
||||
}));
|
||||
const alpha = {
|
||||
id: 'alpha', label: 'Alpha',
|
||||
match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: url.pathname.split('/').pop(), title: 'Alpha', url: value } : null; },
|
||||
async discover() { return { chapters: [] }; }, async run() { return null; },
|
||||
};
|
||||
const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; };
|
||||
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
||||
const first = await service.enqueueDiscoveryChecks('test');
|
||||
const second = await service.enqueueDiscoveryChecks('test');
|
||||
assert.deepEqual(first, { queued: 1, skipped: 0 });
|
||||
assert.deepEqual(second, { queued: 0, skipped: 1 });
|
||||
assert.equal(service.state.queue[0].kind, 'discovery');
|
||||
assert.equal(service.state.queue[0].libraryMangaId, 'Ongoing Manga');
|
||||
});
|
||||
|
||||
test('discovery promotes only chapters newer than the library frontier', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-run-'));
|
||||
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
||||
const chapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0001 [Alpha]');
|
||||
const laterChapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0003 [Alpha]');
|
||||
await fs.mkdir(chapter, { recursive: true });
|
||||
await fs.mkdir(laterChapter, { recursive: true });
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.writeFile(path.join(libraryRoot, 'Ongoing Manga', 'manga.json'), JSON.stringify({
|
||||
title: 'Ongoing Manga', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }],
|
||||
'x-scraper': { ids: { alpha: '42' } },
|
||||
}));
|
||||
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/one' } }));
|
||||
await fs.writeFile(path.join(laterChapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/three' } }));
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
||||
}));
|
||||
let downloadedScope = null;
|
||||
const alpha = {
|
||||
id: 'alpha', label: 'Alpha',
|
||||
match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: '42', title: 'Alpha', url: value } : null; },
|
||||
async discover() { return { chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }, { id: 'three', number: '3' }, { id: 'four', number: '4' }] }; },
|
||||
async run(item, options) { downloadedScope = item.scope; await options.onPublished({ chapter: '4' }); return { finalPath: path.join(libraryRoot, 'Ongoing Manga'), publishedCount: 1, selectedCount: 1 }; },
|
||||
};
|
||||
const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; };
|
||||
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init();
|
||||
const waitFor = async predicate => {
|
||||
const deadline = Date.now() + 3000;
|
||||
while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for discovery'); await new Promise(resolveWait => setTimeout(resolveWait, 20)); }
|
||||
};
|
||||
try {
|
||||
await service.enqueueDiscoveryChecks('test');
|
||||
await service.control('resume');
|
||||
await waitFor(() => service.state.queue[0]?.status === 'complete');
|
||||
assert.deepEqual(downloadedScope, { mode: 'ids', ids: ['four'] });
|
||||
assert.equal(service.state.queue[0].publishedCount, 1);
|
||||
const summary = (await service.publicState()).updateChecks.find(item => item.mangaId === 'Ongoing Manga' && item.source === 'alpha');
|
||||
assert.ok(summary.lastCheckedAt && summary.lastNewContentAt && summary.lastDownloadedAt);
|
||||
assert.equal(summary.lastResult, 'new-content');
|
||||
} finally {
|
||||
await service.close();
|
||||
await fs.rm(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('source-aware scheduler runs one item per source in parallel', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-sources-'));
|
||||
const dataRoot = path.join(temp, 'data');
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
const queue = [
|
||||
{ id: 'A1', source: 'alpha', title: 'Alpha One', url: 'https://alpha.test/1', status: 'queued', order: 0 },
|
||||
{ id: 'A2', source: 'alpha', title: 'Alpha Two', url: 'https://alpha.test/2', status: 'queued', order: 1 },
|
||||
{ id: 'B1', source: 'beta', title: 'Beta One', url: 'https://beta.test/1', status: 'queued', order: 2 },
|
||||
];
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 2, paused: true, status: 'paused', settings: validateSettings({}), queue, history: [], publishedRevision: 0,
|
||||
}));
|
||||
const resolvers = new Map(), started = [];
|
||||
const fakeAdapter = id => ({
|
||||
id, label: id,
|
||||
match(value) { const url = new URL(value); return url.hostname === `${id}.test` ? { source: id, id: url.pathname.slice(1), title: id, url: value } : null; },
|
||||
run(item) { started.push(item.id); return new Promise(resolve => resolvers.set(item.id, () => resolve(path.join(temp, item.id)))); },
|
||||
});
|
||||
const fakeAdapters = [fakeAdapter('alpha'), fakeAdapter('beta')];
|
||||
const resolveAdapter = value => {
|
||||
for (const adapter of fakeAdapters) { const info = adapter.match(value); if (info) return { adapter, info }; }
|
||||
return null;
|
||||
};
|
||||
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot, adapterFor: resolveAdapter, adapters: fakeAdapters }).init();
|
||||
const waitFor = async predicate => {
|
||||
const deadline = Date.now() + 3000;
|
||||
while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for scheduler'); await new Promise(resolve => setTimeout(resolve, 20)); }
|
||||
};
|
||||
try {
|
||||
await service.control('resume');
|
||||
await waitFor(() => resolvers.has('A1') && resolvers.has('B1'));
|
||||
assert.deepEqual(new Set(started), new Set(['A1', 'B1']));
|
||||
assert.equal(resolvers.has('A2'), false);
|
||||
resolvers.get('A1')();
|
||||
await waitFor(() => resolvers.has('A2'));
|
||||
assert.equal(service.state.queue.find(item => item.id === 'A2').status, 'downloading');
|
||||
resolvers.get('A2')(); resolvers.get('B1')();
|
||||
await waitFor(() => service.state.queue.every(item => item.status === 'complete'));
|
||||
assert.equal(service.state.publishedRevision, 3);
|
||||
} finally {
|
||||
for (const resolve of resolvers.values()) resolve();
|
||||
await service.close();
|
||||
await fs.rm(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
const fs = require('fs'), path = require('path'), http = require('http');
|
||||
const { secret } = require('../runtime/config');
|
||||
const { Monitor } = require('./monitor');
|
||||
const root = path.resolve(process.env.SENTINEL_DATA || path.join(process.env.DATA || 'data', 'sentinel'));
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
const token = secret('SCRAPER_TOKEN');
|
||||
const host = process.env.SENTINEL_HOST || '127.0.0.1', port = +(process.env.SENTINEL_PORT || 8092);
|
||||
if (!['127.0.0.1', 'localhost', '::1'].includes(host) && !token) throw new Error('SCRAPER_TOKEN is required for non-loopback Sentinel');
|
||||
const reader = process.env.READER_URL || 'http://127.0.0.1:8080';
|
||||
const scraper = process.env.SCRAPER_URL || 'http://127.0.0.1:8091';
|
||||
const file = path.join(root, 'state.json');
|
||||
let saved = {};
|
||||
if (fs.existsSync(file)) saved = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
const monitor = new Monitor({ incidents: saved.incidents || [] });
|
||||
const request = async (base, endpoint, body) => {
|
||||
const response = await fetch(base + endpoint, { headers: { ...(token ? { authorization: `Bearer ${token}` } : {}), 'content-type': 'application/json' },
|
||||
signal: AbortSignal.timeout(8000), ...(body ? { method: 'POST', body: JSON.stringify(body) } : {}) });
|
||||
if (!response.ok) throw new Error(`Service HTTP ${response.status}`);
|
||||
return response.json();
|
||||
};
|
||||
let stopped = false, timer, lastTick = Date.now();
|
||||
async function tick() {
|
||||
try {
|
||||
const results = await Promise.allSettled([request(reader, '/health'), request(scraper, '/api/state')]);
|
||||
const state = await monitor.observe(results[0].status === 'fulfilled' ? results[0].value : null,
|
||||
results[1].status === 'fulfilled' ? results[1].value : null, input => request(scraper, '/api/recovery', input));
|
||||
fs.writeFileSync(file + '.tmp', JSON.stringify(state, null, 2) + '\n'); fs.renameSync(file + '.tmp', file);
|
||||
lastTick = Date.now();
|
||||
for (const incident of state.incidents.filter(i => i.state === 'open' && !i.logged)) {
|
||||
console.log(JSON.stringify({ ...incident, targetService: incident.service, service: 'sentinel', level: 'warning' })); incident.logged = true;
|
||||
}
|
||||
} catch (error) { console.error('[sentinel]', error.message); }
|
||||
if (!stopped) timer = setTimeout(tick, 30000);
|
||||
}
|
||||
const server = http.createServer((req, res) => {
|
||||
const endpoint = new URL(req.url, 'http://localhost').pathname;
|
||||
let code = 200, value;
|
||||
if (endpoint === '/health') { code = Date.now() - lastTick < 120000 ? 200 : 503; value = { ok: code === 200, service: 'sentinel', checkedAt: monitor.state.checkedAt }; }
|
||||
else if (token && req.headers.authorization !== `Bearer ${token}`) { code = 401; value = { error: 'Unauthorized' }; }
|
||||
else if (endpoint === '/api/state' && req.method === 'GET') value = monitor.state;
|
||||
else if (endpoint === '/metrics' && req.method === 'GET') {
|
||||
res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4' });
|
||||
return res.end(`mangacircus_reader_up ${Number(!!monitor.state.services.reader)}\nmangacircus_scraper_up ${Number(!!monitor.state.services.scraper)}\nmangacircus_open_incidents ${monitor.incidents.filter(i => i.state === 'open').length}\n`);
|
||||
} else { code = 404; value = { error: 'Not found' }; }
|
||||
res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(value));
|
||||
});
|
||||
server.listen(port, host, () => console.log(JSON.stringify({ service: 'sentinel', event: 'started', port })));
|
||||
tick();
|
||||
for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => { stopped = true; clearTimeout(timer); server.close(() => process.exit(0)); });
|
||||
@@ -0,0 +1,62 @@
|
||||
'use strict';
|
||||
const { progressKey } = require('../scraper/lib/failures');
|
||||
|
||||
// No source requests, filesystem checkpoint edits, shell execution or Docker socket.
|
||||
class Monitor {
|
||||
constructor(options = {}) {
|
||||
this.now = options.now || Date.now;
|
||||
this.stallMs = options.stallMs || 900000;
|
||||
this.observations = new Map();
|
||||
this.incidents = options.incidents || [];
|
||||
this.state = { status: 'starting', services: {}, incidents: this.incidents };
|
||||
}
|
||||
incident(key, message, extra = {}) {
|
||||
let incident = this.incidents.find(i => i.key === key && i.state === 'open');
|
||||
if (!incident) {
|
||||
incident = { key, message, state: 'open', at: new Date(this.now()).toISOString(), ...extra };
|
||||
this.incidents.push(incident);
|
||||
if (this.incidents.length > 200) this.incidents.splice(0, this.incidents.length - 200);
|
||||
}
|
||||
return incident;
|
||||
}
|
||||
resolve(key) {
|
||||
for (const i of this.incidents) if (i.key === key && i.state === 'open') { i.state = 'resolved'; i.resolvedAt = new Date(this.now()).toISOString(); }
|
||||
}
|
||||
async observe(reader, scraper, recover) {
|
||||
const now = this.now();
|
||||
this.state.checkedAt = new Date(now).toISOString();
|
||||
this.state.services = { reader: !!reader?.ok, scraper: !!scraper };
|
||||
if (reader?.ready === false) this.incident('reader:scan', 'Reader is alive but its library scan failed; inspect storage and scan diagnostics', { service: 'reader' });
|
||||
else if (reader?.ok) this.resolve('reader:scan');
|
||||
for (const [service, healthy] of Object.entries(this.state.services)) {
|
||||
if (healthy) this.resolve(`service:${service}`);
|
||||
else this.incident(`service:${service}`, `${service} is unreachable or unhealthy; inspect its supervisor`, { service });
|
||||
}
|
||||
if (!scraper) { this.state.status = 'degraded'; return this.state; }
|
||||
const ids = new Set();
|
||||
for (const entry of scraper.active || []) {
|
||||
const id = entry.current.id; ids.add(id);
|
||||
const key = progressKey(entry.job), prior = this.observations.get(id);
|
||||
if (!prior || prior.key !== key) { this.observations.set(id, { key, at: now }); this.resolve(`stall:${id}`); continue; }
|
||||
if (scraper.paused || scraper.sourcePaused?.[entry.source]) continue;
|
||||
const allowance = Math.max(this.stallMs, Number(entry.job?.rate?.chapterMaxMs || 0) + 180000);
|
||||
if (now - prior.at < allowance) continue;
|
||||
const incident = this.incident(`stall:${id}`, 'Job progress has stopped; recovery is limited to one checkpoint-safe attempt', { jobId: id, source: entry.source, title: entry.current.title });
|
||||
if (!incident.attempted && recover) {
|
||||
// Persist attempted in the daemon via stopReason/stallRecoveries; monitor records audit too.
|
||||
incident.attempted = true;
|
||||
try { incident.recovery = await recover({ id, progressKey: key }); }
|
||||
catch { incident.recovery = { changed: false, error: 'Recovery API unavailable' }; }
|
||||
}
|
||||
}
|
||||
for (const id of this.observations.keys()) if (!ids.has(id)) { this.observations.delete(id); this.resolve(`stall:${id}`); }
|
||||
const failures = (scraper.queue || []).filter(item => ['error', 'quarantined'].includes(item.status));
|
||||
for (const item of failures) this.incident(`job:${item.id}`, item.failure?.message || item.error || 'Job needs attention', { jobId: item.id, source: item.source, kind: item.failure?.kind || 'unknown' });
|
||||
for (const i of this.incidents) if (i.key.startsWith('job:') && !failures.some(item => i.key === `job:${item.id}`)) this.resolve(i.key);
|
||||
this.state.sourcePaused = scraper.sourcePaused || {};
|
||||
this.state.active = (scraper.active || []).map(e => ({ source: e.source, id: e.current.id, title: e.current.title, completed: e.job?.completed, total: e.job?.total, updated: e.job?.updated }));
|
||||
this.state.status = this.incidents.some(i => i.state === 'open') ? 'attention' : scraper.paused ? 'paused' : 'healthy';
|
||||
return this.state;
|
||||
}
|
||||
}
|
||||
module.exports = { Monitor };
|
||||
@@ -0,0 +1,26 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const { Monitor } = require('../monitor');
|
||||
test('Sentinel does not disturb progressing or deliberately paused jobs', async () => {
|
||||
let now = 0, recoveries = 0;
|
||||
const monitor = new Monitor({ now: () => now, stallMs: 100 });
|
||||
const state = { paused: false, active: [{ source: 'a', current: { id: 'job', title: 'Manga' }, job: { completed: 0, current: { downloadedPages: 1 } } }], queue: [] };
|
||||
const recover = async () => { recoveries++; return { changed: true }; };
|
||||
await monitor.observe({ ok: true }, state, recover);
|
||||
now = 500000; state.active[0].job.current.downloadedPages = 2;
|
||||
await monitor.observe({ ok: true }, state, recover);
|
||||
now = 1000000; state.paused = true;
|
||||
await monitor.observe({ ok: true }, state, recover);
|
||||
assert.equal(recoveries, 0);
|
||||
state.paused = false; await monitor.observe({ ok: true }, state, recover); await monitor.observe({ ok: true }, state, recover);
|
||||
assert.equal(recoveries, 1);
|
||||
state.active[0].job.completed = 1; await monitor.observe({ ok: true }, state, recover);
|
||||
assert.equal(monitor.incidents.find(i => i.key === 'stall:job').state, 'resolved');
|
||||
});
|
||||
test('Sentinel reports service failures without restart loops and resolves recovery', async () => {
|
||||
const monitor = new Monitor();
|
||||
await monitor.observe(null, null); await monitor.observe(null, null);
|
||||
assert.equal(monitor.incidents.length, 2);
|
||||
await monitor.observe({ ok: true }, { active: [], queue: [], paused: false });
|
||||
assert.equal(monitor.state.status, 'healthy'); assert.ok(monitor.incidents.every(i => i.state === 'resolved'));
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('fs'), os = require('os'), path = require('path'), { EventEmitter } = require('events');
|
||||
const { Supervisor } = require('../../runtime/supervisor');
|
||||
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
test('supervisor isolates a child crash and persists its bounded restart budget', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-supervisor-')), file = path.join(dir, 'runtime.json');
|
||||
let starts = 0;
|
||||
const spawn = () => { const child = new EventEmitter(); child.pid = ++starts; child.kill = () => { child.emit('exit', 0); child.emit('close', 0); }; return child; };
|
||||
const supervisor = new Supervisor('reader', { file, spawn, retryBaseMs: 1 });
|
||||
try {
|
||||
supervisor.start();
|
||||
for (let n = 0; n < 3; n++) {
|
||||
supervisor.child.emit('close', 1, null);
|
||||
for (let i = 0; i < 50 && supervisor.state.status === 'retry_wait'; i++) await delay(2);
|
||||
}
|
||||
assert.equal(starts, 3); assert.equal(supervisor.state.status, 'parked'); assert.equal(supervisor.child, null);
|
||||
const restored = new Supervisor('reader', { file, spawn, retryBaseMs: 1 }); restored.start();
|
||||
assert.equal(starts, 3); assert.equal(restored.state.status, 'parked'); await restored.close();
|
||||
} finally { await supervisor.close(); fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
test('intentional supervisor shutdown does not consume the crash budget', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-supervisor-stop-'));
|
||||
const child = new EventEmitter(); child.pid = 1; child.kill = () => { child.emit('exit', 0); child.emit('close', 0); };
|
||||
const supervisor = new Supervisor('scraper', { file: path.join(dir, 'runtime.json'), spawn: () => child });
|
||||
try { supervisor.start(); await supervisor.close(); assert.equal(supervisor.state.status, 'stopped'); assert.equal(supervisor.state.failures.length, 0); }
|
||||
finally { await supervisor.close(); fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -7,9 +7,16 @@
|
||||
* SIGNUP=closed|open (the very first account can always be created and becomes admin)
|
||||
* SCAN_INTERVAL=10 minutes between automatic library rescans, 0 = off
|
||||
* COOKIE_SECURE=1 mark the session cookie Secure (when served over HTTPS)
|
||||
* SCRAPER_URL=http://127.0.0.1:8091 scraper daemon API
|
||||
* SCRAPER_TOKEN=... shared daemon token (optional on loopback)
|
||||
*/
|
||||
const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||
const { scanLibrary } = require('./library');
|
||||
const { Catalog, CatalogError, collectionViews, TYPES, ROLES } = require('./catalog');
|
||||
const { secret } = require('./runtime/config');
|
||||
const {
|
||||
applyPulse, applyVisit, chapterInsight, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary,
|
||||
} = require('./reading-analytics');
|
||||
|
||||
const PORT = +process.env.PORT || 8080;
|
||||
const LIB = path.resolve(process.env.LIBRARY || 'library');
|
||||
@@ -17,6 +24,9 @@ const DATA = path.resolve(process.env.DATA || 'data');
|
||||
const OPEN_SIGNUP = process.env.SIGNUP === 'open';
|
||||
const SCAN_MINUTES = +(process.env.SCAN_INTERVAL || 10);
|
||||
const SECURE_COOKIE = process.env.COOKIE_SECURE === '1';
|
||||
const SCRAPER_URL = String(process.env.SCRAPER_URL || 'http://127.0.0.1:8091').replace(/\/$/, '');
|
||||
const SCRAPER_TOKEN = secret('SCRAPER_TOKEN');
|
||||
const SENTINEL_URL = String(process.env.SENTINEL_URL || 'http://127.0.0.1:8092').replace(/\/$/, '');
|
||||
const PUB = path.join(__dirname, 'public');
|
||||
const IMG = /\.(jpe?g|png|webp|gif|avif|bmp)$/i;
|
||||
const MIME = {
|
||||
@@ -27,8 +37,9 @@ const MIME = {
|
||||
|
||||
// ---------- storage: one JSON file, written atomically ----------
|
||||
fs.mkdirSync(DATA, { recursive: true });
|
||||
const catalog = new Catalog(DATA);
|
||||
const DB_FILE = path.join(DATA, 'db.json'), CACHE_FILE = path.join(DATA, 'scan-cache.json');
|
||||
const db = { users: {}, sessions: {}, progress: {}, follows: {}, notes: {}, votes: {}, known: {} };
|
||||
const db = { users: {}, sessions: {}, progress: {}, reading: {}, follows: {}, notes: {}, votes: {}, known: {} };
|
||||
// A corrupt db.json stops the server instead of silently starting empty and overwriting it.
|
||||
if (fs.existsSync(DB_FILE)) Object.assign(db, JSON.parse(fs.readFileSync(DB_FILE, 'utf8')));
|
||||
function writeAtomic(file, text) { fs.writeFileSync(file + '.tmp', text); fs.renameSync(file + '.tmp', file); }
|
||||
@@ -48,7 +59,7 @@ function checkPw(pw, stored) {
|
||||
return want.length === got.length && crypto.timingSafeEqual(want, got);
|
||||
}
|
||||
const DUMMY_HASH = hashPw('x');
|
||||
const DEFAULTS = { startMode: 'last', lastMode: 'page', adapt: true, skipExtras: false, skipCovers: false, zoom: {}, webhook: '' };
|
||||
const DEFAULTS = { startMode: 'last', lastMode: 'page', adapt: true, skipExtras: false, skipCovers: false, flowPageGap: true, readingClock: false, readingStory: true, zoom: {}, webhook: '' };
|
||||
const settingsOf = u => ({ ...DEFAULTS, ...u.settings, zoom: { ...(u.settings && u.settings.zoom) } });
|
||||
|
||||
function createUser(name, pw, admin, replace) {
|
||||
@@ -126,8 +137,27 @@ const url = rel => (rel ? 'lib/' + rel.split('/').map(encodeURIComponent).join('
|
||||
const mangaOf = id => lib.byId.get(String(id)) || bad(404, 'Manga not found');
|
||||
const chapterOf = (m, id) => m.chapterById.get(String(id)) || bad(404, 'Chapter not found');
|
||||
const progressOf = (u, m) => (db.progress[u.name] && db.progress[u.name][m.id]) || {};
|
||||
const mangaReadingOf = (u, m) => (db.reading[u.name] && db.reading[u.name][m.id]) || {};
|
||||
const readingOf = (u, m, ch) => (db.reading[u.name] && db.reading[u.name][m.id] && db.reading[u.name][m.id][ch.id]) || null;
|
||||
const unreadOf = u => (db.notes[u.name] || []).filter(n => !n.read).length;
|
||||
const admin = u => u.admin || bad(403, 'Admins only');
|
||||
const READING_LEASE_MS = 15000;
|
||||
const readingLeases = new Map();
|
||||
|
||||
async function scraperRequest(endpoint, body) {
|
||||
const headers = { accept: 'application/json' };
|
||||
if (SCRAPER_TOKEN) headers.authorization = `Bearer ${SCRAPER_TOKEN}`;
|
||||
const options = { headers, signal: AbortSignal.timeout(7000) };
|
||||
if (body !== undefined) {
|
||||
options.method = 'POST'; headers['content-type'] = 'application/json'; options.body = JSON.stringify(body);
|
||||
}
|
||||
let response;
|
||||
try { response = await fetch(SCRAPER_URL + endpoint, options); }
|
||||
catch (error) { throw new HttpError(503, `Scraper service unavailable: ${error.message}`); }
|
||||
const data = await response.json().catch(() => ({}));
|
||||
if (!response.ok) throw new HttpError(response.status >= 500 ? 502 : response.status, data.error || `Scraper HTTP ${response.status}`);
|
||||
return data;
|
||||
}
|
||||
|
||||
// Versions sorted by the sum of all user votes; ties go to the higher "priority" in chapter.json, then folder order.
|
||||
function ranked(m, ch) {
|
||||
@@ -142,6 +172,28 @@ function versionsOf(m, ch, u) {
|
||||
id: v.id, source: v.source, sourceUrl: v.sourceUrl, pages: v.pages.length, added: v.added, score, my: (votes[v.id] && votes[v.id][u.name]) || 0,
|
||||
}));
|
||||
}
|
||||
|
||||
// Normalize the release-date shapes accepted by manga.json for library sorting.
|
||||
function firstReleaseOf(meta) {
|
||||
meta = meta || {};
|
||||
const raw = meta.firstReleaseDate || meta.releaseDate || meta.startDate;
|
||||
let year, month, day, label, time;
|
||||
if (raw && typeof raw === 'object') {
|
||||
year = +raw.year; month = +raw.month || 1; day = +raw.day || 1;
|
||||
if (year > 0) {
|
||||
label = [String(year).padStart(4, '0'), raw.month ? String(month).padStart(2, '0') : null, raw.day ? String(day).padStart(2, '0') : null].filter(Boolean).join('-');
|
||||
time = Date.UTC(year, month - 1, day);
|
||||
}
|
||||
} else if (typeof raw === 'string' && raw.trim()) {
|
||||
const parsed = Date.parse(raw);
|
||||
if (Number.isFinite(parsed)) { label = raw.trim(); time = parsed; }
|
||||
}
|
||||
if (!Number.isFinite(time)) {
|
||||
year = +meta.year;
|
||||
if (year > 0) { label = String(year); time = Date.UTC(year, 0, 1); }
|
||||
}
|
||||
return { label: label || '', time: Number.isFinite(time) ? time : 0 };
|
||||
}
|
||||
function continueOf(m, prog, s) {
|
||||
const ok = c => !(s.skipExtras && c.extra);
|
||||
let last = null;
|
||||
@@ -162,6 +214,48 @@ function neighbour(m, ch, step, s) {
|
||||
// ---------- API ----------
|
||||
const PUBLIC = new Set(['/api/auth', '/api/login', '/api/signup']);
|
||||
const routes = {
|
||||
'GET /api/health': () => ({ ok: !scanState.error, service: 'reader', uptime: Math.round(process.uptime()), scan: scanState }),
|
||||
'GET /api/collections': ({ u }) => ({ collections: routes['GET /api/library']({ u }).collections }),
|
||||
'GET /api/collection': ({ u, q }) => {
|
||||
const view = routes['GET /api/library']({ u }).collections.find(c => c.id === q.get('id')) || bad(404, 'Collection not found');
|
||||
const next = view.members.find(m => m.read < m.chapters) || view.members[0];
|
||||
const manga = mangaOf(next.id);
|
||||
return { ...view, continue: { m: next.id, ...continueOf(manga, progressOf(u, manga), settingsOf(u)) },
|
||||
relations: catalog.state.relations.filter(r => r.state === 'confirmed' && view.members.some(m => m.id === r.from || m.id === r.to)) };
|
||||
},
|
||||
'GET /api/admin/catalog': ({ u }) => { admin(u); return { ...catalog.state, types: TYPES, roles: ROLES, mangas: lib.mangas.map(m => ({ id: m.id, title: m.title })) }; },
|
||||
'POST /api/admin/catalog': ({ u, b }) => { admin(u); return catalog.update(b, new Set(lib.byId.keys()), u.name); },
|
||||
'POST /api/admin/catalog/suggest': ({ u, b }) => {
|
||||
admin(u);
|
||||
return catalog.update({ ...catalog.state, revision: b.revision, relations: catalog.suggest(lib.mangas) }, new Set(lib.byId.keys()), u.name);
|
||||
},
|
||||
'POST /api/admin/catalog/research': async ({ u, b }) => {
|
||||
admin(u);
|
||||
const m = mangaOf(b.m), id = m.meta?.['x-anilist']?.id;
|
||||
if (!id) bad(400, 'Assign a verified AniList identity first');
|
||||
const { lookupAniListById } = require('./scraper/metadata/anilist');
|
||||
const meta = await lookupAniListById(id, { cacheDir: path.join(DATA, 'metadata', 'anilist') });
|
||||
const enriched = lib.mangas.map(item => item.id === m.id ? { ...item, meta } : item);
|
||||
return catalog.update({ ...catalog.state, revision: b.revision, relations: catalog.suggest(enriched) }, new Set(lib.byId.keys()), u.name);
|
||||
},
|
||||
'POST /api/admin/collection-cover': ({ u, b }) => {
|
||||
admin(u);
|
||||
const c = catalog.state.collections.find(c => c.id === b.id) || bad(404, 'Collection not found');
|
||||
if (b.revision !== catalog.state.revision) bad(409, 'Catalog changed; reload before saving');
|
||||
const image = Buffer.from(String(b.base64 || ''), 'base64');
|
||||
const ext = require('./scraper/lib/images').imageExtension(image);
|
||||
if (!ext || image.length > 600000) bad(400, 'Use a JPG, PNG, WebP, GIF, AVIF or BMP under 600 KB');
|
||||
const name = sha(image) + '.' + ext;
|
||||
const dir = path.join(DATA, 'collection-covers'); fs.mkdirSync(dir, { recursive: true });
|
||||
if (!fs.existsSync(path.join(dir, name))) fs.writeFileSync(path.join(dir, name), image, { flag: 'wx' });
|
||||
return catalog.update({ ...catalog.state, collections: catalog.state.collections.map(item => item.id === c.id ? { ...item, coverMode: 'custom', customCover: name } : item) }, new Set(lib.byId.keys()), u.name);
|
||||
},
|
||||
'GET /api/admin/health': async ({ u }) => {
|
||||
admin(u);
|
||||
const sentinel = await fetch(SENTINEL_URL + '/api/state', { headers: SCRAPER_TOKEN ? { authorization: `Bearer ${SCRAPER_TOKEN}` } : {}, signal: AbortSignal.timeout(4000) })
|
||||
.then(async r => { if (!r.ok) throw new Error(); return r.json(); }).catch(() => ({ status: 'unreachable' }));
|
||||
return { reader: { ok: !scanState.error, uptime: Math.round(process.uptime()), scan: scanState }, sentinel };
|
||||
},
|
||||
'GET /api/auth': () => ({ firstRun: !Object.keys(db.users).length, signup: OPEN_SIGNUP }),
|
||||
|
||||
'POST /api/signup': ({ b, res }) => {
|
||||
@@ -189,7 +283,7 @@ const routes = {
|
||||
const s = (u.settings = u.settings || {});
|
||||
if (['last', 'flow', 'page', 'book'].includes(b.startMode)) s.startMode = b.startMode;
|
||||
if (['flow', 'page', 'book'].includes(b.lastMode)) s.lastMode = b.lastMode;
|
||||
for (const k of ['adapt', 'skipExtras', 'skipCovers']) if (typeof b[k] === 'boolean') s[k] = b[k];
|
||||
for (const k of ['adapt', 'skipExtras', 'skipCovers', 'flowPageGap', 'readingClock', 'readingStory']) if (typeof b[k] === 'boolean') s[k] = b[k];
|
||||
if (b.zoom && typeof b.zoom === 'object') {
|
||||
s.zoom = {};
|
||||
for (const k of ['flow', 'page', 'book', 'strip']) if (+b.zoom[k] > 0) s.zoom[k] = Math.min(8, Math.max(0.1, +b.zoom[k]));
|
||||
@@ -206,33 +300,85 @@ const routes = {
|
||||
|
||||
'GET /api/library': ({ u }) => {
|
||||
const prog = db.progress[u.name] || {}, fol = db.follows[u.name] || {}, notes = (db.notes[u.name] || []).filter(n => !n.read);
|
||||
return {
|
||||
scanning: !!scanning && !lib.mangas.length,
|
||||
mangas: lib.mangas.map(m => {
|
||||
const p = prog[m.id] || {};
|
||||
const showStory = settingsOf(u).readingStory;
|
||||
const mangas = lib.mangas.map(m => {
|
||||
const p = prog[m.id] || {}, meta = m.meta || {}, release = firstReleaseOf(meta);
|
||||
return {
|
||||
id: m.id, title: m.title, cover: url(m.cover), direction: m.direction, chapters: m.chapters.length, updated: m.updated,
|
||||
altTitles: Array.isArray(meta.altTitles) ? meta.altTitles : (meta.altTitles ? [String(meta.altTitles)] : []),
|
||||
authors: Array.isArray(meta.authors) ? meta.authors : (meta.authors ? [String(meta.authors)] : []),
|
||||
tags: Array.isArray(meta.tags) ? meta.tags : (meta.tags ? [String(meta.tags)] : []), status: meta.status || '', year: meta.year || '',
|
||||
firstRelease: release.label, firstReleaseAt: release.time,
|
||||
read: m.chapters.filter(c => p[c.id] && p[c.id].done).length,
|
||||
lastRead: Object.values(p).reduce((t, x) => Math.max(t, x.t || 0), 0),
|
||||
following: !!fol[m.id], unread: notes.filter(n => n.m === m.id).length,
|
||||
reading: showStory ? mangaInsight(mangaReadingOf(u, m), m.chapters.map(ch => ch.id)) : null,
|
||||
};
|
||||
}),
|
||||
});
|
||||
return {
|
||||
scanning: !!scanning && !lib.mangas.length,
|
||||
reading: showStory ? libraryInsight(mangas) : null,
|
||||
mangas,
|
||||
collections: collectionViews(catalog.state, mangas),
|
||||
};
|
||||
},
|
||||
|
||||
'GET /api/manga': ({ u, q }) => {
|
||||
const m = mangaOf(q.get('m')), prog = progressOf(u, m);
|
||||
const m = mangaOf(q.get('m')), prog = progressOf(u, m), showStory = settingsOf(u).readingStory;
|
||||
const records = mangaReadingOf(u, m);
|
||||
return {
|
||||
id: m.id, title: m.title, meta: m.meta, cover: url(m.cover), direction: m.direction,
|
||||
collections: catalog.state.collections.filter(c => c.members.some(member => member.mangaId === m.id)).map(c => ({ id: c.id, title: c.title })),
|
||||
relations: catalog.state.relations.filter(r => r.state === 'confirmed' && (r.from === m.id || r.to === m.id)).map(r => ({ ...r,
|
||||
otherId: r.from === m.id ? r.to : r.from, otherTitle: lib.byId.get(r.from === m.id ? r.to : r.from)?.title || (r.from === m.id ? r.to : r.from) })),
|
||||
following: !!(db.follows[u.name] && db.follows[u.name][m.id]),
|
||||
books: m.books.map(b => ({ key: b.key, label: b.label, title: b.title, extra: b.extra })),
|
||||
chapters: m.chapters.map(ch => ({
|
||||
id: ch.id, book: ch.book, label: ch.label, title: ch.title, extra: ch.extra, progress: prog[ch.id] || null, versions: versionsOf(m, ch, u),
|
||||
reading: showStory ? chapterInsight(records[ch.id]) : null,
|
||||
})),
|
||||
reading: showStory ? mangaInsight(records, m.chapters.map(ch => ch.id)) : null,
|
||||
continue: continueOf(m, prog, settingsOf(u)),
|
||||
};
|
||||
},
|
||||
|
||||
'GET /api/reading/export': ({ u }) => {
|
||||
const byManga = db.reading[u.name] || {};
|
||||
const mangas = Object.entries(byManga).map(([id, records]) => {
|
||||
const m = lib.byId.get(id), chapterIds = m ? m.chapters.map(ch => ch.id) : Object.keys(records || {});
|
||||
const chapterMap = m ? m.chapterById : new Map();
|
||||
return {
|
||||
id, title: m ? m.title : id,
|
||||
reading: mangaInsight(records, chapterIds),
|
||||
chapters: chapterIds.map(chapterId => {
|
||||
const chapter = chapterMap.get(chapterId), reading = chapterInsight(records && records[chapterId]);
|
||||
return reading ? { id: chapterId, label: chapter ? chapter.label : chapterId, title: chapter ? chapter.title : '', reading } : null;
|
||||
}).filter(Boolean),
|
||||
};
|
||||
}).filter(item => item.reading.timedChapters || item.reading.completedPasses);
|
||||
return {
|
||||
schema: 1,
|
||||
exportedAt: new Date().toISOString(),
|
||||
user: u.name,
|
||||
reading: libraryInsight(mangas),
|
||||
mangas,
|
||||
};
|
||||
},
|
||||
|
||||
'POST /api/reading/delete': ({ u, b }) => {
|
||||
if (b.all) {
|
||||
if (!checkPw(String(b.password || ''), u.hash)) bad(403, 'The current password is wrong.');
|
||||
delete db.reading[u.name];
|
||||
readingLeases.delete(u.name);
|
||||
save();
|
||||
return { ok: true, deleted: 'all' };
|
||||
}
|
||||
const m = mangaOf(b.m), all = db.reading[u.name];
|
||||
if (all) delete all[m.id];
|
||||
save();
|
||||
return { ok: true, deleted: m.id };
|
||||
},
|
||||
|
||||
'GET /api/chapter': ({ u, q }) => {
|
||||
const m = mangaOf(q.get('m')), ch = chapterOf(m, q.get('c')), s = settingsOf(u), prog = progressOf(u, m);
|
||||
const v = ch.versions.find(x => x.id === q.get('v')) || ranked(m, ch)[0].v;
|
||||
@@ -247,6 +393,7 @@ const routes = {
|
||||
pages: v.pages.map(p => ({ u: url(v.rel + '/' + p.f), w: p.w, h: p.h, t: p.t, r: p.r })),
|
||||
prev: neighbour(m, ch, -1, s), next: neighbour(m, ch, 1, s),
|
||||
progress: prog[ch.id] || null,
|
||||
reading: s.readingClock ? publicSummary(readingOf(u, m, ch), v.id, eligiblePages(v, s)) : null,
|
||||
// for the chapter picker: the whole manga across all books
|
||||
books: m.books.map(b => ({ key: b.key, label: b.label, title: b.title })),
|
||||
chapters: m.chapters.map(c => ({ id: c.id, book: c.book, label: c.label, title: c.title, extra: c.extra, done: !!(prog[c.id] && prog[c.id].done) })),
|
||||
@@ -263,6 +410,51 @@ const routes = {
|
||||
save();
|
||||
return { ok: true };
|
||||
},
|
||||
// Separate endpoint: an older server on rollback cannot misread cumulative
|
||||
// visit totals as legacy deltas. Old open tabs retain their original protocol.
|
||||
'POST /api/reading/visit': context => routes['POST /api/reading/pulse']({ ...context, cumulative: true }),
|
||||
'POST /api/reading/pulse': ({ u, b, cumulative = false }) => {
|
||||
const settings = settingsOf(u);
|
||||
if (!settings.readingClock) bad(403, 'The Active Reading Clock is disabled.');
|
||||
const m = mangaOf(b.m), ch = chapterOf(m, b.c);
|
||||
const v = ch.versions.find(x => x.id === b.v) || bad(404, 'Version not found');
|
||||
const session = String(b.session || '');
|
||||
const seq = Number(b.seq);
|
||||
if (!/^[a-zA-Z0-9_-]{8,80}$/.test(session)) bad(400, 'Invalid reading session.');
|
||||
if (!Number.isInteger(seq) || seq < 1 || seq > 1e9) bad(400, 'Invalid reading pulse sequence.');
|
||||
|
||||
const now = Date.now(), eligible = eligiblePages(v, settings), lease = readingLeases.get(u.name);
|
||||
const previous = readingOf(u, m, ch)?.recentSessions?.find(item => item.id === session);
|
||||
// A stale request must not reclaim a lease after a newer exit save. Final
|
||||
// snapshots can also arrive before the initial claim: do not lose that visit.
|
||||
if (cumulative && previous && seq <= previous.seq) return {
|
||||
accepted: true, duplicate: true, leaseMs: 0,
|
||||
reading: publicSummary(readingOf(u, m, ch), v.id, eligible),
|
||||
};
|
||||
const lateFinal = cumulative && b.final;
|
||||
if (!lateFinal && lease && lease.session !== session && now - lease.at < READING_LEASE_MS) {
|
||||
return {
|
||||
accepted: false,
|
||||
reason: 'other-tab',
|
||||
retryAfterMs: READING_LEASE_MS - (now - lease.at),
|
||||
reading: publicSummary(readingOf(u, m, ch), v.id, eligible),
|
||||
};
|
||||
}
|
||||
if (!lateFinal) readingLeases.set(u.name, { session, at: now });
|
||||
|
||||
const byUser = (db.reading[u.name] = db.reading[u.name] || {});
|
||||
const byManga = (byUser[m.id] = byUser[m.id] || {});
|
||||
const record = (byManga[ch.id] = readingOf(u, m, ch) || emptyRecord());
|
||||
const result = (cumulative ? applyVisit : applyPulse)(record, b, { version: v.id, pageCount: v.pages.length, eligible, now });
|
||||
if (b.release && readingLeases.get(u.name)?.session === session) readingLeases.delete(u.name);
|
||||
if (!result.duplicate) save();
|
||||
return {
|
||||
accepted: true,
|
||||
duplicate: result.duplicate,
|
||||
leaseMs: b.release ? 0 : READING_LEASE_MS,
|
||||
reading: publicSummary(record, v.id, eligible),
|
||||
};
|
||||
},
|
||||
'POST /api/unread': ({ u, b }) => {
|
||||
const m = mangaOf(b.m), mp = db.progress[u.name] && db.progress[u.name][m.id];
|
||||
if (mp) delete mp[chapterOf(m, b.c).id];
|
||||
@@ -311,11 +503,34 @@ const routes = {
|
||||
admin(u);
|
||||
const name = String(b.name || '');
|
||||
if (name === u.name) bad(400, "You can't delete your own account.");
|
||||
delete db.users[name]; delete db.progress[name]; delete db.follows[name]; delete db.notes[name];
|
||||
delete db.users[name]; delete db.progress[name]; delete db.reading[name]; delete db.follows[name]; delete db.notes[name];
|
||||
for (const [k, s] of Object.entries(db.sessions)) if (s.u === name) delete db.sessions[k];
|
||||
save();
|
||||
return { ok: true };
|
||||
},
|
||||
'GET /api/admin/scraper': async ({ u }) => {
|
||||
admin(u);
|
||||
try { return { available: true, ...(await scraperRequest('/api/state')) }; }
|
||||
catch (error) { return { available: false, error: error.message, url: SCRAPER_URL }; }
|
||||
},
|
||||
'POST /api/admin/scraper': async ({ u, b }) => {
|
||||
admin(u);
|
||||
const command = String(b.command || '');
|
||||
if (command === 'enqueue') return scraperRequest('/api/queue', {
|
||||
url: b.url, title: b.title, metadataSearch: b.metadataSearch,
|
||||
scope: b.scope, existingPolicy: b.existingPolicy, libraryMangaId: b.libraryMangaId,
|
||||
inspect: b.inspect === true,
|
||||
});
|
||||
if (command === 'remove') return scraperRequest('/api/queue/remove', { id: b.id });
|
||||
if (command === 'clear-completed') return scraperRequest('/api/queue/clear-completed', { source: b.source });
|
||||
if (command === 'reorder') return scraperRequest('/api/queue/reorder', { ids: b.ids });
|
||||
if (command === 'control') return scraperRequest('/api/control', { action: b.action, id: b.id });
|
||||
if (command === 'settings') return scraperRequest('/api/settings', b.settings || {});
|
||||
if (command === 'discover') return scraperRequest('/api/discovery/run', {});
|
||||
if (command === 'auto-update') return scraperRequest('/api/library/auto-update', { id: b.id, enabled: !!b.enabled });
|
||||
if (command === 'publish-legacy-ready') return scraperRequest('/api/legacy/publish-ready', {});
|
||||
bad(400, 'Unknown scraper command');
|
||||
},
|
||||
};
|
||||
|
||||
// ---------- HTTP ----------
|
||||
@@ -355,6 +570,13 @@ http.createServer(async (req, res) => {
|
||||
try {
|
||||
const { pathname, searchParams } = new URL(req.url, 'http://localhost');
|
||||
const u = userOf(req);
|
||||
if (pathname === '/health' && req.method === 'GET') return sendJson(res, 200, { ...routes['GET /api/health'](), ok: true, ready: !scanState.error });
|
||||
if (pathname === '/api/collection-cover' && req.method === 'GET') {
|
||||
if (!u) bad(401, 'Please log in');
|
||||
const c = catalog.state.collections.find(c => c.id === searchParams.get('id'));
|
||||
if (!c?.customCover) bad(404, 'Cover not found');
|
||||
return serveFile(req, res, path.join(DATA, 'collection-covers'), c.customCover, 'private, no-cache', IMG);
|
||||
}
|
||||
if (pathname.startsWith('/api/')) {
|
||||
const route = routes[req.method + ' ' + pathname];
|
||||
if (!route) bad(404, 'Unknown endpoint');
|
||||
@@ -368,10 +590,25 @@ http.createServer(async (req, res) => {
|
||||
}
|
||||
serveFile(req, res, PUB, pathname === '/' ? 'index.html' : pathname.slice(1), 'no-cache');
|
||||
} catch (e) {
|
||||
if (!(e instanceof HttpError)) console.error(e);
|
||||
if (!res.headersSent) sendJson(res, e.code || 500, { error: e instanceof HttpError ? e.message : 'Internal error' });
|
||||
const expected = e instanceof HttpError || e instanceof CatalogError;
|
||||
if (!expected) console.error(e);
|
||||
if (!res.headersSent) sendJson(res, expected ? e.code : 500, { error: expected ? e.message : 'Internal error' });
|
||||
}
|
||||
}).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`));
|
||||
|
||||
rescan();
|
||||
if (SCAN_MINUTES > 0) setInterval(() => rescan(), SCAN_MINUTES * 60000).unref();
|
||||
|
||||
// A completed scraper publish increments a durable revision. Polling it here keeps the
|
||||
// reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.
|
||||
let scraperRevision = null;
|
||||
async function pollScraper() {
|
||||
try {
|
||||
const state = await scraperRequest('/api/state');
|
||||
const revision = Number(state.publishedRevision || 0);
|
||||
if (scraperRevision !== null && revision > scraperRevision) await rescan();
|
||||
scraperRevision = revision;
|
||||
} catch { /* the reader also works when the optional scraper daemon is offline */ }
|
||||
}
|
||||
pollScraper();
|
||||
setInterval(pollScraper, 15000).unref();
|
||||
@@ -0,0 +1,32 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict'), fs = require('fs'), os = require('os'), path = require('path');
|
||||
const { Catalog, normalize, collectionViews } = require('../catalog');
|
||||
const input = () => ({ collections: [{ id: 'franchise', title: 'Franchise', members: [{ mangaId: 'a', role: 'main', readingOrder: 1 }, { mangaId: 'b', role: 'sequel', readingOrder: 2 }] }], relations: [{ from: 'a', to: 'b', type: 'sequel', state: 'confirmed', evidence: [] }] });
|
||||
test('collections persist without changing manga identities and reject stale edits', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-catalog-'));
|
||||
try {
|
||||
const c = new Catalog(dir); c.update({ ...input(), revision: 0 }, new Set(['a', 'b']), 'admin');
|
||||
assert.equal(new Catalog(dir).state.revision, 1);
|
||||
assert.throws(() => c.update({ ...input(), revision: 0 }, new Set(['a', 'b']), 'admin'), { code: 409 });
|
||||
assert.deepEqual(c.state.collections[0].members.map(m => m.mangaId), ['a', 'b']);
|
||||
assert.throws(() => normalize({ ...input(), collections: [...input().collections, { id: 'other', title: 'Other', members: [{ mangaId: 'a' }] }] }, new Set(['a', 'b'])), /already belongs/);
|
||||
assert.throws(() => normalize({ ...input(), relations: [{ from: 'a', to: 'a', type: 'sequel' }] }, new Set(['a', 'b'])), /Invalid relation/);
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
test('collections aggregate progress once and preserve absent members', () => {
|
||||
const normalized = normalize(input(), new Set(['a', 'b']));
|
||||
const views = collectionViews(normalized, [{ id: 'a', title: 'A', chapters: 3, read: 2, unread: 1, tags: ['Fantasy'], status: 'completed', reading: { activeMs: 30 } }, { id: 'b', title: 'B', chapters: 2, read: 1, unread: 1, tags: ['Fantasy'], status: 'ongoing', reading: { activeMs: 20 } }]);
|
||||
assert.equal(views[0].chapters, 5); assert.equal(views[0].read, 3); assert.equal(views[0].reading.activeMs, 50);
|
||||
assert.equal(views[0].status, 'ongoing'); assert.deepEqual(views[0].tags, ['Fantasy']);
|
||||
assert.equal(normalize(normalized, new Set(['a']), normalized).collections[0].members.length, 2);
|
||||
});
|
||||
test('metadata suggestions retain manual and rejected decisions', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-suggestions-'));
|
||||
try {
|
||||
const c = new Catalog(dir);
|
||||
const mangas = [{ id: 'a', meta: { 'x-anilist': { id: 10, relations: [{ id: 11, type: 'SEQUEL' }] } } }, { id: 'b', meta: { 'x-anilist': { id: 11 } } }];
|
||||
const relations = c.suggest(mangas); assert.equal(relations.length, 1); assert.equal(relations[0].state, 'suggested');
|
||||
c.update({ revision: 0, collections: [], relations: [{ ...relations[0], state: 'rejected' }] }, new Set(['a', 'b']), 'admin');
|
||||
assert.equal(c.suggest(mangas).length, 1); assert.equal(c.suggest(mangas)[0].state, 'rejected');
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('node:fs'), path = require('node:path'), vm = require('node:vm');
|
||||
const crypto = require('node:crypto');
|
||||
const source = fs.readFileSync(path.join(__dirname, '../public/collections.js'), 'utf8').split('(async () => {')[0];
|
||||
test('Collection IDs use the native secure-context API when available', () => {
|
||||
const context = { crypto: { randomUUID: () => 'native-uuid' } };
|
||||
vm.runInNewContext(source, context);
|
||||
assert.equal(context.collectionId(), 'native-uuid');
|
||||
});
|
||||
test('Collection IDs work on HTTP without randomUUID and remain unique valid UUIDs', () => {
|
||||
const context = { crypto: { getRandomValues: values => crypto.randomFillSync(values) } };
|
||||
vm.runInNewContext(source, context);
|
||||
const ids = Array.from({ length: 200 }, () => context.collectionId());
|
||||
assert.equal(new Set(ids).size, ids.length);
|
||||
for (const id of ids) assert.match(id, /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/);
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const {
|
||||
applyPulse, chapterInsight, compactRanges, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary,
|
||||
} = require('../reading-analytics');
|
||||
|
||||
const pulse = (session, seq, extra = {}) => ({
|
||||
session, seq, activeMs: 0, loadingMs: 0,
|
||||
inputMs: { mouse: 0, touch: 0, keyboard: 0 }, seen: [],
|
||||
fastBursts: 0, rejectedPageViews: 0, maxPps: 0,
|
||||
...extra,
|
||||
});
|
||||
|
||||
test('page ranges remain compact and deterministic', () => {
|
||||
assert.deepEqual(compactRanges([8, 2, 3, 3, 4, 9, 12]), [[2, 4], [8, 9], [12, 12]]);
|
||||
});
|
||||
|
||||
test('reading pulses are idempotent and server-clamped', () => {
|
||||
const record = emptyRecord(), options = { version: 'source-a', pageCount: 4, eligible: [0, 1, 2, 3], now: 10000 };
|
||||
applyPulse(record, pulse('session-one', 1, { activeMs: 9800, inputMs: { touch: 9800 } }), options);
|
||||
assert.equal(record.totalActiveMs, 9800);
|
||||
assert.equal(record.sessions, 1);
|
||||
assert.equal(record.inputMs.touch, 9800);
|
||||
|
||||
const duplicate = applyPulse(record, pulse('session-one', 1, { activeMs: 9800 }), { ...options, now: 20000 });
|
||||
assert.equal(duplicate.duplicate, true);
|
||||
assert.equal(record.totalActiveMs, 9800);
|
||||
|
||||
applyPulse(record, pulse('session-one', 2, { activeMs: 50000, loadingMs: 50000, inputMs: { mouse: 50000 } }), { ...options, now: 21000 });
|
||||
assert.equal(record.totalActiveMs + record.loadingMs, 9800 + 12000);
|
||||
assert.ok(record.inputMs.mouse <= 6000);
|
||||
});
|
||||
|
||||
test('eligible coverage completes a pass and a reread starts a fresh pass', () => {
|
||||
const version = { pages: [{ t: ['cover'] }, { t: [] }, { t: ['extra'] }, { t: [] }] };
|
||||
const eligible = eligiblePages(version, { skipCovers: true, skipExtras: true });
|
||||
assert.deepEqual(eligible, [1, 3]);
|
||||
|
||||
const record = emptyRecord();
|
||||
applyPulse(record, pulse('first-pass', 1, { activeMs: 1000, inputMs: { keyboard: 1000 }, seen: [0, 1, 2, 3] }), {
|
||||
version: 'source-a', pageCount: 4, eligible, now: 1000,
|
||||
});
|
||||
assert.equal(record.completedPasses, 1);
|
||||
assert.equal(record.firstCompleteMs, 1000);
|
||||
assert.deepEqual(record.currentPass.seenRanges, [[1, 1], [3, 3]]);
|
||||
|
||||
applyPulse(record, pulse('second-pass', 1), { version: 'source-a', pageCount: 4, eligible, now: 2000 });
|
||||
const summary = publicSummary(record, 'source-a', eligible);
|
||||
assert.equal(summary.currentPass.complete, false);
|
||||
assert.equal(summary.currentPass.seenPages, 0);
|
||||
assert.equal(summary.totalActiveMs, 1000);
|
||||
});
|
||||
|
||||
test('switching source versions starts separate page coverage without losing lifetime time', () => {
|
||||
const record = emptyRecord();
|
||||
applyPulse(record, pulse('version-test', 1, { activeMs: 800, seen: [0] }), {
|
||||
version: 'source-a', pageCount: 2, eligible: [0, 1], now: 1000,
|
||||
});
|
||||
applyPulse(record, pulse('version-test', 2, { activeMs: 500, seen: [2] }), {
|
||||
version: 'source-b', pageCount: 3, eligible: [0, 1, 2], now: 2000,
|
||||
});
|
||||
const summary = publicSummary(record, 'source-b', [0, 1, 2]);
|
||||
assert.equal(summary.totalActiveMs, 1300);
|
||||
assert.deepEqual(summary.currentPass.seenRanges, [[2, 2]]);
|
||||
});
|
||||
|
||||
test('chapter and manga insights distinguish untimed, partial, complete and reread records', () => {
|
||||
assert.equal(chapterInsight(null), null);
|
||||
const records = {
|
||||
first: { ...emptyRecord(), totalActiveMs: 60000, updatedAt: 100, currentPass: { activeMs: 60000 } },
|
||||
second: { ...emptyRecord(), totalActiveMs: 540000, lastCompleteMs: 180000, firstCompleteMs: 210000, completedPasses: 2, updatedAt: 300, currentPass: { activeMs: 180000, completedAt: 300 } },
|
||||
third: { ...emptyRecord(), totalActiveMs: 300000, lastCompleteMs: 300000, firstCompleteMs: 300000, completedPasses: 1, updatedAt: 200, currentPass: { activeMs: 300000, completedAt: 200 } },
|
||||
};
|
||||
assert.deepEqual(chapterInsight(records.first), {
|
||||
totalActiveMs: 60000, firstCompleteMs: null, lastCompleteMs: null, completedPasses: 0,
|
||||
currentPassMs: 60000, currentPassComplete: false, updatedAt: 100,
|
||||
});
|
||||
assert.deepEqual(mangaInsight(records, ['first', 'second', 'third', 'missing']), {
|
||||
activeMs: 900000,
|
||||
timedChapters: 3,
|
||||
completedChapters: 2,
|
||||
completedPasses: 3,
|
||||
rereads: 1,
|
||||
typicalChapterMs: 240000,
|
||||
lastChapterId: 'second',
|
||||
updatedAt: 300,
|
||||
});
|
||||
});
|
||||
|
||||
test('library insights select the most-read and most-recent manga', () => {
|
||||
const summary = libraryInsight([
|
||||
{ id: 'a', reading: { activeMs: 1000, timedChapters: 1, completedPasses: 1, rereads: 0, updatedAt: 100, lastChapterId: '1/1' } },
|
||||
{ id: 'b', reading: { activeMs: 3000, timedChapters: 2, completedPasses: 4, rereads: 2, updatedAt: 90, lastChapterId: '1/2' } },
|
||||
{ id: 'empty', reading: { activeMs: 0, timedChapters: 0, completedPasses: 0, rereads: 0, updatedAt: 0 } },
|
||||
]);
|
||||
assert.deepEqual(summary, {
|
||||
activeMs: 4000, timedChapters: 3, completedPasses: 5, rereads: 2, timedMangas: 2,
|
||||
mostTimeMangaId: 'b', lastMangaId: 'a', lastChapterId: '1/1', updatedAt: 100,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const { applyVisit, emptyRecord, chapterInsight, publicSummary } = require('../reading-analytics');
|
||||
const { exposureTracker, stateFor, IDLE_MS } = require('../public/reading-clock-core');
|
||||
const pages = n => Array.from({ length: n }, (_, i) => i);
|
||||
const config = { version: 'webtoon', pageCount: 20, eligible: pages(20), now: 10000 };
|
||||
const visit = (extra = {}) => ({ session: 'reading-visit', seq: 1, elapsedMs: 10000, activeMs: 9000, seen: [], ...extra });
|
||||
|
||||
test('59% is a draft; 60% credits the entire visit and final duration keeps growing', () => {
|
||||
const r = emptyRecord(), cfg = { ...config, pageCount: 100, eligible: pages(100) };
|
||||
applyVisit(r, visit({ seen: pages(59) }), cfg);
|
||||
assert.equal(r.totalActiveMs, 0); assert.equal(chapterInsight(r), null);
|
||||
applyVisit(r, visit({ seq: 2, elapsedMs: 12000, activeMs: 11000, seen: pages(60) }), cfg);
|
||||
assert.equal(r.totalActiveMs, 11000); assert.equal(r.completedPasses, 1);
|
||||
applyVisit(r, visit({ seq: 3, elapsedMs: 18000, activeMs: 16000, seen: pages(65), final: true }), cfg);
|
||||
assert.equal(r.totalActiveMs, 16000); assert.equal(r.firstCompleteMs, 16000); assert.equal(r.lastCompleteMs, 16000);
|
||||
assert.equal(r.currentPass.closed, true);
|
||||
});
|
||||
test('an exit snapshot overtaking a pulse counts once and survives a JSON reload', () => {
|
||||
const r = emptyRecord();
|
||||
applyVisit(r, visit({ seq: 3, seen: pages(12), final: true }), config);
|
||||
const restored = JSON.parse(JSON.stringify(r));
|
||||
assert.equal(applyVisit(restored, visit({ seq: 1, activeMs: 0 }), config).duplicate, true);
|
||||
assert.equal(applyVisit(restored, visit({ seq: 3, seen: pages(12), final: true }), config).duplicate, true);
|
||||
applyVisit(restored, visit({ seq: 4, seen: pages(13), activeMs: 10000, elapsedMs: 11000 }), config);
|
||||
assert.equal(restored.totalActiveMs, 10000); assert.equal(restored.completedPasses, 1);
|
||||
});
|
||||
test('delayed delivery does not truncate an honest minute to twelve seconds', () => {
|
||||
const r = emptyRecord();
|
||||
applyVisit(r, visit({ elapsedMs: 65000, activeMs: 60000, inputMs: { touch: 60000 }, seen: pages(12) }), config);
|
||||
assert.equal(r.totalActiveMs, 60000); assert.equal(r.inputMs.touch, 60000);
|
||||
});
|
||||
test('one-picture lookup does not borrow prior coverage or change saved statistics', () => {
|
||||
const r = emptyRecord();
|
||||
applyVisit(r, visit({ seen: pages(12), final: true }), config);
|
||||
applyVisit(r, visit({ session: 'lookup-visit', elapsedMs: 60000, activeMs: 59000, seen: [15], final: true }), { ...config, now: 70000 });
|
||||
assert.equal(r.totalActiveMs, 9000); assert.equal(r.completedPasses, 1);
|
||||
assert.equal(chapterInsight(r).updatedAt, 10000);
|
||||
});
|
||||
test('late finals update their own visit without replacing a newer visit', () => {
|
||||
const r = emptyRecord();
|
||||
applyVisit(r, visit({ seen: pages(12) }), config);
|
||||
applyVisit(r, visit({ session: 'new-visit', seen: pages(12), activeMs: 4000 }), { ...config, now: 20000 });
|
||||
applyVisit(r, visit({ seq: 2, seen: pages(14), elapsedMs: 15000, activeMs: 13000, final: true }), { ...config, now: 21000 });
|
||||
assert.equal(r.currentPass.visit, 'new-visit'); assert.equal(r.firstCompleteMs, 13000);
|
||||
assert.equal(r.lastCompleteMs, 4000); assert.equal(r.totalActiveMs, 17000);
|
||||
});
|
||||
test('legacy time and source IDs are retained; covers/extras use the eligible denominator', () => {
|
||||
const r = { ...emptyRecord(), totalActiveMs: 5000, completedPasses: 1, firstCompleteMs: 5000, lastCompleteMs: 5000, updatedAt: 500 };
|
||||
const cfg = { ...config, eligible: [1, 3, 5, 7, 9] };
|
||||
applyVisit(r, visit({ seen: [0, 1, 3, 5, 19], activeMs: 2000 }), cfg);
|
||||
assert.equal(r.totalActiveMs, 7000); assert.equal(r.firstCompleteMs, 5000);
|
||||
assert.equal(publicSummary(r, 'webtoon', cfg.eligible).currentPass.requiredPages, 3);
|
||||
});
|
||||
test('fast webtoon slices above five pages per second are credited without a ceiling', () => {
|
||||
const t = exposureTracker();
|
||||
for (let i = 0; i < 10; i++) t.update([{ index: i, loaded: true }], i * 120, true);
|
||||
t.update([], 1200, true);
|
||||
assert.deepEqual([...t.seen], pages(10));
|
||||
});
|
||||
test('jumping over pages to dwell on one image never invents skipped coverage', () => {
|
||||
const t = exposureTracker();
|
||||
t.update([{ index: 0, loaded: true }], 0, true);
|
||||
t.update([{ index: 19, loaded: true }], 50, true);
|
||||
t.update([{ index: 19, loaded: true }], 60000, true);
|
||||
assert.deepEqual([...t.seen], [19]);
|
||||
});
|
||||
test('unloaded and hidden pages cannot qualify; mixed loading does not pause readable pages', () => {
|
||||
assert.equal(IDLE_MS, 30000, 'The user-selected inactivity allowance is 30 seconds');
|
||||
const t = exposureTracker();
|
||||
t.update([{ index: 0, loaded: false }], 0, true);
|
||||
t.update([{ index: 0, loaded: false }], 1000, true);
|
||||
assert.equal(t.seen.size, 0);
|
||||
const view = { pages: [{ index: 0, loaded: true }, { index: 1, loaded: false }] };
|
||||
const s = { visible: true, focused: true, lastInputAt: 0, now: 5000, owned: true, view };
|
||||
assert.equal(stateFor(s), 'reading');
|
||||
assert.equal(stateFor({ ...s, visible: false }), 'hidden');
|
||||
assert.equal(stateFor({ ...s, focused: false }), 'unfocused');
|
||||
assert.equal(stateFor({ ...s, now: IDLE_MS }), 'idle');
|
||||
assert.equal(stateFor({ ...s, now: IDLE_MS - 1 }), 'reading');
|
||||
t.update(view.pages, 1000, false); t.update(view.pages, 100000, false);
|
||||
assert.equal(t.seen.size, 0);
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('fs'), os = require('os'), path = require('path'), net = require('net');
|
||||
const { spawn } = require('child_process');
|
||||
const { Canvas } = require('../tools/png');
|
||||
const root = path.resolve(__dirname, '..');
|
||||
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
async function port() {
|
||||
const server = net.createServer();
|
||||
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
||||
const result = server.address().port; await new Promise(resolve => server.close(resolve)); return result;
|
||||
}
|
||||
test('three services integrate collections, auth, custom covers and private health without source traffic', { timeout: 30000 }, async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-services-'));
|
||||
const children = [], output = [];
|
||||
try {
|
||||
const data = path.join(dir, 'data'), library = path.join(dir, 'library');
|
||||
fs.mkdirSync(path.join(data, 'scraper'), { recursive: true });
|
||||
fs.writeFileSync(path.join(data, 'scraper', 'queue.json'), JSON.stringify({ paused: true, queue: [], settings: { autoUpdateEnabled: false } }));
|
||||
const png = new Canvas(32, 48, [10, 90, 130]).png();
|
||||
for (const name of ['Main Work', 'Sequel']) {
|
||||
const folder = path.join(library, name, 'v01', 'c001'); fs.mkdirSync(folder, { recursive: true });
|
||||
fs.writeFileSync(path.join(folder, '001.png'), png);
|
||||
fs.writeFileSync(path.join(library, name, 'cover.png'), png);
|
||||
fs.writeFileSync(path.join(library, name, 'manga.json'), JSON.stringify({ title: name, tags: ['Fantasy'], status: 'ongoing' }));
|
||||
}
|
||||
const [rp, sp, wp] = await Promise.all([port(), port(), port()]);
|
||||
const base = `http://127.0.0.1:${rp}`, scraper = `http://127.0.0.1:${sp}`, sentinel = `http://127.0.0.1:${wp}`;
|
||||
const token = 'integration-test-only';
|
||||
const env = { ...process.env, DATA: data, LIBRARY: library, SCRAPER_DATA: path.join(data, 'scraper'), SENTINEL_DATA: path.join(data, 'sentinel'),
|
||||
PORT: String(rp), SCRAPER_PORT: String(sp), SENTINEL_PORT: String(wp), SCRAPER_HOST: '127.0.0.1', SENTINEL_HOST: '127.0.0.1',
|
||||
READER_URL: base, SCRAPER_URL: scraper, SENTINEL_URL: sentinel, SCRAPER_TOKEN: token, SCRAPER_TOKEN_FILE: '', SCAN_INTERVAL: '0', COOKIE_SECURE: '0' };
|
||||
for (const script of ['server.js', 'scraper/daemon.js', 'sentinel/daemon.js']) {
|
||||
const child = spawn(process.execPath, [script], { cwd: root, env, windowsHide: true }); children.push(child);
|
||||
for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => output.push(chunk.toString()));
|
||||
}
|
||||
for (const url of [base, scraper, sentinel]) {
|
||||
let ready = false;
|
||||
for (let n = 0; n < 80 && !ready; n++) { ready = await fetch(url + '/health').then(r => r.ok).catch(() => false); if (!ready) await delay(100); }
|
||||
assert.ok(ready, output.join(''));
|
||||
}
|
||||
assert.equal((await fetch(scraper + '/api/state')).status, 401);
|
||||
assert.equal((await fetch(sentinel + '/api/state')).status, 401);
|
||||
const signup = await fetch(base + '/api/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'tester', password: 'local-test-password' }) });
|
||||
assert.equal(signup.status, 200); const cookie = signup.headers.get('set-cookie').split(';')[0];
|
||||
const request = (endpoint, body) => fetch(base + endpoint, { headers: { cookie, 'content-type': 'application/json' }, ...(body ? { method: 'POST', body: JSON.stringify(body) } : {}) });
|
||||
let inventory;
|
||||
for (let n = 0; n < 50; n++) { inventory = await (await request('/api/library')).json(); if (inventory.mangas.length === 2) break; await delay(100); }
|
||||
assert.equal(inventory.mangas.length, 2);
|
||||
const ids = inventory.mangas.map(m => m.id);
|
||||
const catalog = { revision: 0, collections: [{ id: 'demo', title: 'Demo collection', members: ids.map((mangaId, index) => ({ mangaId, readingOrder: index + 1, role: index ? 'sequel' : 'main' })), coverMode: 'mosaic' }], relations: [{ from: ids[0], to: ids[1], type: 'sequel', state: 'confirmed' }] };
|
||||
let response = await request('/api/admin/catalog', catalog); assert.equal(response.status, 200); const saved = await response.json();
|
||||
response = await request('/api/admin/catalog', catalog); assert.equal(response.status, 409); assert.match((await response.json()).error, /reload/);
|
||||
const view = await (await request('/api/collection?id=demo')).json(); assert.equal(view.members.length, 2); assert.equal(view.chapters, 2); assert.equal(view.relations.length, 1);
|
||||
let revision = saved.revision;
|
||||
for (let n = 0; n < 2; n++) {
|
||||
const cover = await (await request('/api/admin/collection-cover', { id: 'demo', revision, base64: png.toString('base64') })).json(); revision = cover.revision;
|
||||
const image = await request('/api/collection-cover?id=demo'); assert.equal(image.status, 200); assert.deepEqual(Buffer.from(await image.arrayBuffer()), png);
|
||||
}
|
||||
const health = await (await request('/api/admin/health')).json(); assert.ok(health.reader); assert.ok(health.sentinel);
|
||||
assert.equal((await request('/api/admin/scraper')).status, 200);
|
||||
// Exercise the actual authenticated visit route and cross-chapter lease races.
|
||||
assert.equal((await request('/api/settings', { readingClock: true })).status, 200);
|
||||
const readerChapter = async m => {
|
||||
const detail = await (await request('/api/manga?m=' + encodeURIComponent(m))).json();
|
||||
return (await request('/api/chapter?m=' + encodeURIComponent(m) + '&c=' + encodeURIComponent(detail.chapters[0].id))).json();
|
||||
};
|
||||
const first = await readerChapter(ids[0]), second = await readerChapter(ids[1]);
|
||||
const visit = (d, session, seq, extra = {}) => ({ m: d.manga.id, c: d.chapter.id, v: d.version, session, seq, elapsedMs: 2000, activeMs: 0, seen: [], ...extra });
|
||||
const send = async b => (await request('/api/reading/visit', b)).json();
|
||||
await send(visit(first, 'first-visit', 1));
|
||||
let result = await send(visit(first, 'first-visit', 3, { activeMs: 1000, seen: [0], final: true, release: true }));
|
||||
assert.equal(result.reading.totalActiveMs, 1000);
|
||||
assert.equal((await send(visit(second, 'second-visit', 1))).accepted, true);
|
||||
// The stale in-flight pulse arrives after exit and after the next reader claims.
|
||||
assert.equal((await send(visit(first, 'first-visit', 2))).duplicate, true);
|
||||
result = await send(visit(first, 'first-visit', 4, { activeMs: 1500, seen: [0], final: true, release: true }));
|
||||
assert.equal(result.accepted, true); assert.equal(result.reading.totalActiveMs, 1500);
|
||||
// A beacon arriving before its first claim is also retained, without stealing
|
||||
// the current reader's lease.
|
||||
assert.equal((await send(visit(first, 'beacon-first', 2, { activeMs: 500, seen: [0], final: true, release: true }))).accepted, true);
|
||||
assert.equal((await send(visit(second, 'third-visit', 1))).accepted, false);
|
||||
await send(visit(second, 'second-visit', 2, { release: true }));
|
||||
assert.equal((await fetch(base + '/api/reading/visit', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(visit(first, 'unauth-visit', 1)) })).status, 401);
|
||||
await request('/api/settings', { readingClock: false });
|
||||
assert.equal((await request('/api/reading/visit', visit(first, 'disabled-visit', 1))).status, 403);
|
||||
const unauth = await fetch(base + '/api/admin/catalog'); assert.equal(unauth.status, 401);
|
||||
await request('/api/signup', { name: 'reader', password: 'local-test-password' });
|
||||
const login = await fetch(base + '/api/login', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'reader', password: 'local-test-password' }) });
|
||||
// Sign-up is closed after first-run; the admin catalog remains protected.
|
||||
assert.equal(login.status, 403);
|
||||
} finally {
|
||||
await Promise.all(children.map(child => new Promise(resolve => { if (child.exitCode !== null) return resolve(); child.once('exit', resolve); child.kill(); })));
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { SETTINGS_SECTIONS, SCRAPER_TABS, settingsRoute } = require('../public/settings-nav');
|
||||
|
||||
test('settings organize unique routes into User and Admin, with scraper-local tabs', () => {
|
||||
assert.equal(new Set(SETTINGS_SECTIONS.map(s => s.id)).size, SETTINGS_SECTIONS.length);
|
||||
assert.deepEqual([...new Set(SETTINGS_SECTIONS.map(s => s.group))], ['User', 'Admin']);
|
||||
assert.ok(SETTINGS_SECTIONS.every(s => s.id.split('/').length === 2));
|
||||
assert.deepEqual(SCRAPER_TABS, ['activity', 'queue', 'updates', 'sources']);
|
||||
for (const section of SETTINGS_SECTIONS) assert.equal(settingsRoute('#' + section.id, true).id, section.id);
|
||||
});
|
||||
|
||||
test('settings deep links default safely and never route ordinary readers into admin', () => {
|
||||
assert.deepEqual(settingsRoute('', false), { id: 'user/reading', tab: 'activity' });
|
||||
assert.equal(settingsRoute('#not/a/section', true).id, 'user/reading');
|
||||
assert.equal(settingsRoute('#admin/users', false).id, 'user/reading');
|
||||
assert.equal(settingsRoute('#admin/scraper/hosters', false).id, 'user/reading');
|
||||
assert.equal(settingsRoute('#user/notifications', false).id, 'user/notifications');
|
||||
assert.deepEqual(settingsRoute('#admin/scraper/hosters', true), { id: 'admin/scraper', tab: 'sources' });
|
||||
assert.deepEqual(settingsRoute('#admin/scraper/sources', true), { id: 'admin/scraper', tab: 'sources' });
|
||||
assert.equal(settingsRoute('#admin/scraper/unknown', true).tab, 'activity');
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
'use strict';
|
||||
// Isolated browser QA. Never reads production user data or contacts image sources.
|
||||
const fs = require('fs'), os = require('os'), path = require('path'), { spawn } = require('child_process');
|
||||
const { Canvas } = require('../tools/png');
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-ui-')), data = path.join(dir, 'data'), library = path.join(dir, 'library');
|
||||
fs.mkdirSync(path.join(data, 'scraper'), { recursive: true });
|
||||
fs.writeFileSync(path.join(data, 'scraper', 'queue.json'), JSON.stringify({ paused: true, queue: [], settings: { autoUpdateEnabled: false } }));
|
||||
for (const [index, name] of ['Moon Garden', 'Moon Garden Before the Rain', 'Iron Harbor'].entries()) {
|
||||
const folder = path.join(library, name, 'v01', 'c001 The Beginning'); fs.mkdirSync(folder, { recursive: true });
|
||||
const png = new Canvas(180, 270, [50 + index * 40, 60, 120]).png();
|
||||
fs.writeFileSync(path.join(folder, '001.png'), png); fs.writeFileSync(path.join(library, name, 'cover.png'), png);
|
||||
fs.writeFileSync(path.join(library, name, 'manga.json'), JSON.stringify({ title: name, status: index ? 'completed' : 'ongoing', tags: ['Fantasy', 'Demo'] }));
|
||||
}
|
||||
const env = { ...process.env, DATA: data, LIBRARY: library, PORT: '8180', SCRAPER_PORT: '8191', SENTINEL_PORT: '8192',
|
||||
READER_URL: 'http://127.0.0.1:8180', SCRAPER_URL: 'http://127.0.0.1:8191', SENTINEL_URL: 'http://127.0.0.1:8192',
|
||||
SCRAPER_DATA: path.join(data, 'scraper'), SENTINEL_DATA: path.join(data, 'sentinel'), COOKIE_SECURE: '0', SCAN_INTERVAL: '0', SCRAPER_TOKEN: 'fixture', SCRAPER_TOKEN_FILE: '' };
|
||||
const child = spawn(process.execPath, ['tools/start-all.js'], { cwd: path.resolve(__dirname, '..'), env, stdio: 'inherit', windowsHide: true });
|
||||
console.log('Isolated UI fixture at http://127.0.0.1:8180 · first account is test admin');
|
||||
for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill('SIGTERM'));
|
||||
child.once('exit', code => { fs.rmSync(dir, { recursive: true, force: true }); process.exit(code || 0); });
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
const { secret } = require('../runtime/config');
|
||||
const token = secret('SCRAPER_TOKEN');
|
||||
const root = process.env.READER_URL || 'http://127.0.0.1:8080';
|
||||
const scraper = process.env.SCRAPER_URL || 'http://127.0.0.1:8091';
|
||||
const sentinel = process.env.SENTINEL_URL || 'http://127.0.0.1:8092';
|
||||
(async () => {
|
||||
for (const [name, base, endpoint] of [['reader', root, '/health'], ['scraper', scraper, '/api/state'], ['sentinel', sentinel, '/api/state']]) {
|
||||
try {
|
||||
const response = await fetch(base + endpoint, { headers: token ? { authorization: `Bearer ${token}` } : {}, signal: AbortSignal.timeout(10000) });
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
const state = await response.json();
|
||||
console.log(JSON.stringify({ service: name, reachable: true, status: state.status || (state.ok ? 'healthy' : 'unhealthy'),
|
||||
active: state.active?.map(e => ({ source: e.source, title: e.current?.title || e.title, completed: e.job?.completed ?? e.completed, total: e.job?.total ?? e.total })),
|
||||
openIncidents: state.incidents?.filter(i => i.state === 'open').length, sourcePaused: state.sourcePaused }));
|
||||
} catch (error) { console.log(JSON.stringify({ service: name, reachable: false, error: error.message })); process.exitCode = 1; }
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// Run on the host. Authentication stays inside the reader container; no token
|
||||
// appears in command arguments, terminal output or a published daemon port.
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const path = require('node:path');
|
||||
const action = process.argv[2] || 'check';
|
||||
const id = process.argv[3];
|
||||
if (!['check', 'state', 'pause', 'resume', 'retry'].includes(action) ||
|
||||
(action === 'retry' && !id) || (id && !/^[a-zA-Z0-9-]{1,80}$/.test(id))) {
|
||||
console.error('Usage: node tools/compose-control.js check|state|pause|resume|retry <job-id>');
|
||||
process.exit(2);
|
||||
}
|
||||
const script = action === 'check' ? "require('./tools/check.js')" : `
|
||||
const {secret}=require('./runtime/config');
|
||||
(async()=>{
|
||||
const action=${JSON.stringify(action)},id=${JSON.stringify(id || null)};
|
||||
const response=await fetch(process.env.SCRAPER_URL+(action==='state'?'/api/state':'/api/control'),{
|
||||
method:action==='state'?'GET':'POST',signal:AbortSignal.timeout(20000),
|
||||
headers:{authorization:'Bearer '+secret('SCRAPER_TOKEN'),'content-type':'application/json'},
|
||||
...(action==='state'?{}:{body:JSON.stringify({action,id})})
|
||||
});
|
||||
if(!response.ok)throw new Error('Scraper returned HTTP '+response.status);
|
||||
const s=await response.json();
|
||||
const job=j=>j?{updated:j.updated,completed:j.completed,total:j.total,current:j.current,lastChapter:j.lastChapter}:null;
|
||||
console.log(JSON.stringify({version:s.version,status:s.status,paused:s.paused,settings:s.settings,
|
||||
sourcePaused:s.sourcePaused,publishedRevision:s.publishedRevision,
|
||||
active:s.active?.map(a=>({source:a.source,id:a.current?.id,title:a.current?.title,job:job(a.job)})),
|
||||
queue:s.queue?.map(q=>({id:q.id,title:q.title,source:q.source,status:q.status,failure:q.failure, nextAttemptAt:q.nextAttemptAt})),
|
||||
openIncidents:s.incidents?.filter(i=>i.state==='open').map(i=>({kind:i.kind,title:i.title,message:i.message})),
|
||||
runtimeHeartbeat:s.runtimeHeartbeat,nextDiscoveryAt:s.nextDiscoveryAt,libraryStatus:s.libraryStatus},null,2));
|
||||
})().catch(e=>{console.error(e.message);process.exitCode=1;});`;
|
||||
const result = spawnSync('docker', ['compose', 'exec', '-T', 'reader', 'node', '-e', script], {
|
||||
cwd: path.resolve(__dirname, '..'), encoding: 'utf8', stdio: 'inherit', windowsHide: true,
|
||||
});
|
||||
if (result.error) console.error(result.error.message);
|
||||
process.exit(result.status ?? 1);
|
||||
@@ -0,0 +1,5 @@
|
||||
'use strict';
|
||||
const ports = { reader: process.env.PORT || 8080, scraper: process.env.SCRAPER_PORT || 8091, sentinel: process.env.SENTINEL_PORT || 8092 };
|
||||
const service = process.argv[2] || 'reader';
|
||||
fetch(`http://127.0.0.1:${ports[service]}/health`, { signal: AbortSignal.timeout(4000) })
|
||||
.then(async r => { const body = await r.json(); if (!r.ok || !body.ok) process.exitCode = 1; }).catch(() => { process.exitCode = 1; });
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
const fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||
const root = path.resolve(__dirname, '..');
|
||||
for (const relative of ['data', 'data/scraper', 'data/sentinel', 'library', '.secrets']) fs.mkdirSync(path.join(root, relative), { recursive: true });
|
||||
const file = path.join(root, '.secrets', 'scraper_token');
|
||||
if (!fs.existsSync(file)) fs.writeFileSync(file, crypto.randomBytes(32).toString('hex') + '\n', { flag: 'wx', mode: 0o600 });
|
||||
if (!fs.existsSync(path.join(root, '.env'))) fs.copyFileSync(path.join(root, '.env.example'), path.join(root, '.env'));
|
||||
console.log('Compose directories and private token are ready. Stop existing writers before using these paths in containers.');
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// Reviewed, one-series migration for ambiguous pre-provenance metadata. Dry run
|
||||
// by default. Apply only during a stopped-writer cutover with a state snapshot.
|
||||
const fs = require('node:fs').promises;
|
||||
const path = require('node:path');
|
||||
const { imageSize } = require('../library');
|
||||
const { detectVerticalStrip, KEY } = require('../reading-direction');
|
||||
const { writeJsonAtomic } = require('../scraper/lib/publisher');
|
||||
|
||||
const [folder, expectedId, ...flags] = process.argv.slice(2);
|
||||
const apply = flags.includes('--apply');
|
||||
const mustInclude = flags.find(flag => flag.startsWith('--chapter='))?.slice('--chapter='.length);
|
||||
if (!folder || !/^[A-Z0-9]{20,40}$/.test(expectedId || '') ||
|
||||
(mustInclude && !/^c\d{1,6}$/i.test(mustInclude)) ||
|
||||
flags.some(flag => !['--apply', '--writers-stopped'].includes(flag) && !/^--chapter=c\d{1,6}$/i.test(flag)) ||
|
||||
(apply && !flags.includes('--writers-stopped'))) {
|
||||
console.error('Usage: node tools/review-reading-direction.js <library-manga-folder> <weebcentral-id> [--chapter=c0056] [--apply --writers-stopped]');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const libraryRoot = path.resolve(process.env.LIBRARY || path.join(__dirname, '..', 'library'));
|
||||
const mangaDir = path.resolve(folder);
|
||||
if (path.dirname(mangaDir) !== libraryRoot) throw new Error('Manga folder must be directly inside the configured library');
|
||||
const file = path.join(mangaDir, 'manga.json');
|
||||
const original = await fs.readFile(file, 'utf8');
|
||||
const meta = JSON.parse(original);
|
||||
if (meta['x-scraper']?.ids?.weebcentral !== expectedId) throw new Error('Source identity did not match');
|
||||
if (meta.direction !== 'rtl' || meta.directionOverride || meta[KEY]) throw new Error('Only an unmarked legacy RTL value can be reviewed');
|
||||
const chapters = [];
|
||||
for (const book of await fs.readdir(mangaDir, { withFileTypes: true })) {
|
||||
if (!book.isDirectory() || !/^v\d/i.test(book.name)) continue;
|
||||
for (const chapter of await fs.readdir(path.join(mangaDir, book.name), { withFileTypes: true })) {
|
||||
if (chapter.isDirectory() && /^c\d/i.test(chapter.name)) chapters.push(path.join(mangaDir, book.name, chapter.name));
|
||||
}
|
||||
}
|
||||
chapters.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
|
||||
if (chapters.length < 3) throw new Error('Too few chapters for reviewed migration');
|
||||
const requiredChapter = mustInclude && chapters.find(item => path.basename(item).toLowerCase().startsWith(mustInclude.toLowerCase() + ' '));
|
||||
if (mustInclude && !requiredChapter) throw new Error('Required chapter was not found');
|
||||
const selected = [...new Set([chapters[0], chapters[Math.floor(chapters.length / 2)], chapters.at(-1),
|
||||
requiredChapter].filter(Boolean))];
|
||||
const results = [];
|
||||
for (const dir of selected) {
|
||||
const pages = [];
|
||||
for (const entry of (await fs.readdir(dir)).sort()) {
|
||||
if (!/\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(entry)) continue;
|
||||
const size = await imageSize(path.join(dir, entry)).catch(() => null);
|
||||
const rotation = /_rot(90|180|270)(?:_|\.)/i.exec(entry);
|
||||
pages.push({ w: size?.w || 0, h: size?.h || 0, r: rotation ? Number(rotation[1]) : 0 });
|
||||
}
|
||||
const detected = detectVerticalStrip(pages);
|
||||
results.push({ chapter: path.basename(dir), detected: detected?.value || null,
|
||||
evidence: detected?.evidence || { pages: pages.length } });
|
||||
}
|
||||
const supporting = results.filter(item => item.detected === 'ttb').length;
|
||||
const reviewed = supporting >= Math.max(3, Math.ceil(results.length * .75)) &&
|
||||
(!requiredChapter || results.some(item => item.chapter === path.basename(requiredChapter) && item.detected === 'ttb'));
|
||||
if (apply) {
|
||||
if (!reviewed) throw new Error('Insufficient consistent strip evidence; metadata unchanged');
|
||||
if (await fs.readFile(file, 'utf8') !== original) throw new Error('Metadata changed during review');
|
||||
await writeJsonAtomic(file, { ...meta, direction: 'ttb',
|
||||
[KEY]: { version: 1, value: 'ttb', source: 'detected',
|
||||
evidence: { reviewedChapters: results.map(item => item.chapter), supportingChapters: supporting } } });
|
||||
}
|
||||
console.log(JSON.stringify({ manga: path.basename(mangaDir), reviewed, applied: apply, samples: results }, null, 2));
|
||||
}
|
||||
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
@@ -0,0 +1,10 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$projectRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..')).Path
|
||||
$nodePath = (Get-Command node -ErrorAction Stop).Source
|
||||
$outputLog = Join-Path $projectRoot 'data\scraper\supervisor.out.log'
|
||||
$errorLog = Join-Path $projectRoot 'data\scraper\supervisor.err.log'
|
||||
|
||||
Set-Location -LiteralPath $projectRoot
|
||||
& $nodePath 'tools/start-all.js' 1>> $outputLog 2>> $errorLog
|
||||
exit $LASTEXITCODE
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// Targeted recovery snapshot, not a library backup. Caller must prove all writers
|
||||
// are stopped first. Never run this concurrently with the reader or scraper.
|
||||
const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto');
|
||||
if (process.argv[2] !== '--writers-stopped') throw new Error('First verify that every writer is stopped, then pass --writers-stopped');
|
||||
const root = path.resolve(__dirname, '..'), library = path.join(root, 'library'), data = path.join(root, 'data');
|
||||
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
|
||||
const backup = path.join(data, 'scraper', 'backups', stamp + '-compose-cutover');
|
||||
fs.mkdirSync(backup, { recursive: true });
|
||||
const files = [];
|
||||
function copy(source, relative) {
|
||||
if (!fs.existsSync(source)) return;
|
||||
if (fs.statSync(source).isDirectory()) {
|
||||
for (const name of fs.readdirSync(source)) copy(path.join(source, name), path.join(relative, name));
|
||||
return;
|
||||
}
|
||||
const target = path.join(backup, relative); fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
fs.copyFileSync(source, target);
|
||||
files.push({ path: relative, bytes: fs.statSync(source).size, sha256: crypto.createHash('sha256').update(fs.readFileSync(source)).digest('hex') });
|
||||
}
|
||||
for (const name of ['db.json', 'scan-cache.json', 'catalog.json', 'collection-covers', 'runtime-reader.json']) copy(path.join(data, name), path.join('data', name));
|
||||
for (const name of ['queue.json', 'runtime.json', 'daemon.lock', 'supervisor.lock']) copy(path.join(data, 'scraper', name), path.join('data', 'scraper', name));
|
||||
for (const name of ['state.json', 'runtime.json']) copy(path.join(data, 'sentinel', name), path.join('data', 'sentinel', name));
|
||||
for (const name of fs.readdirSync(library)) {
|
||||
copy(path.join(library, name, name.startsWith('.mcscraper-work-') ? 'job.json' : 'manga.json'), path.join('library', name, name.startsWith('.mcscraper-work-') ? 'job.json' : 'manga.json'));
|
||||
}
|
||||
fs.writeFileSync(path.join(backup, 'snapshot.json'), JSON.stringify({ created: new Date().toISOString(), kind: 'targeted-state-not-image-backup', files }, null, 2));
|
||||
console.log(JSON.stringify({ backup, files: files.length, imageFilesCopied: false }));
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs'), path = require('path');
|
||||
const { Supervisor } = require('../runtime/supervisor');
|
||||
const root = path.resolve(__dirname, '..');
|
||||
const lock = path.resolve(process.env.SCRAPER_DATA || path.join(process.env.DATA || path.join(root, 'data'), 'scraper'), 'supervisor.lock');
|
||||
fs.mkdirSync(path.dirname(lock), { recursive: true });
|
||||
if (fs.existsSync(lock)) {
|
||||
const pid = Number(fs.readFileSync(lock, 'utf8'));
|
||||
try { process.kill(pid, 0); throw new Error(`Supervisor already alive (PID ${pid})`); }
|
||||
catch (e) { if (e.code !== 'ESRCH') throw e; fs.unlinkSync(lock); }
|
||||
}
|
||||
fs.writeFileSync(lock, String(process.pid), { flag: 'wx' });
|
||||
const supervisors = ['reader', 'scraper', 'sentinel'].map(name => new Supervisor(name, { root }));
|
||||
supervisors.forEach(service => service.start());
|
||||
const keepalive = setInterval(() => {}, 60000);
|
||||
let closing = false;
|
||||
for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, async () => {
|
||||
if (closing) return; closing = true; clearInterval(keepalive);
|
||||
await Promise.all(supervisors.map(service => service.close()));
|
||||
fs.unlinkSync(lock); process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,243 @@
|
||||
'use strict';
|
||||
// Real browser, disposable Compose storage; never uses a production account/catalog.
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs'), path = require('node:path');
|
||||
module.exports = async function testCollectionsUI(url, cookie) {
|
||||
const { chromium } = require('playwright');
|
||||
const browser = await chromium.launch({ channel: process.env.QA_BROWSER_CHANNEL || 'msedge', headless: true });
|
||||
const origin = 'http://manga-qa.test:' + new URL(url).port;
|
||||
const context = await browser.newContext({ viewport: { width: 1280, height: 900 } });
|
||||
const [name, ...value] = cookie.split('=');
|
||||
await context.addCookies([{ name, value: value.join('='), url: origin }]);
|
||||
let delaySave = false, failSave = false, saveCount = 0;
|
||||
// Proxy only this synthetic origin to the local disposable fixture. The browser
|
||||
// still sees ordinary non-loopback HTTP, as it does on a Tailscale IP address.
|
||||
await context.route(origin + '/**', async route => {
|
||||
if (new URL(route.request().url()).pathname === '/api/admin/catalog' && route.request().method() === 'POST') {
|
||||
saveCount++;
|
||||
if (failSave) { failSave = false; return route.fulfill({ status: 503, contentType: 'application/json', body: JSON.stringify({ error: 'Fixture temporary save failure' }) }); }
|
||||
if (delaySave) await new Promise(resolve => setTimeout(resolve, 600));
|
||||
}
|
||||
const response = await route.fetch({ url: url + route.request().url().slice(origin.length) });
|
||||
await route.fulfill({ response });
|
||||
});
|
||||
const page = await context.newPage(), errors = [];
|
||||
page.on('pageerror', e => errors.push(e.message));
|
||||
const shots = path.resolve(__dirname, '..', 'data', 'qa', 'collections');
|
||||
fs.mkdirSync(shots, { recursive: true });
|
||||
const catalog = () => page.evaluate(() => fetch('/api/admin/catalog').then(r => r.json()));
|
||||
const status = page.getByRole('status');
|
||||
const save = async (button = 'Save collection') => {
|
||||
await page.getByRole('button', { name: button, exact: true }).click();
|
||||
await page.waitForFunction(() => document.querySelector('[role=status]')?.textContent === 'Saved');
|
||||
};
|
||||
const noOverflow = async () => assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), 'Collections page has horizontal overflow');
|
||||
try {
|
||||
await page.goto(origin + '/collections.html');
|
||||
await page.getByRole('button', { name: 'New collection', exact: true }).waitFor();
|
||||
assert.equal(await page.evaluate(() => isSecureContext), false, 'QA must reproduce an insecure browser origin');
|
||||
assert.equal(await page.evaluate(() => typeof crypto.randomUUID), 'undefined');
|
||||
await page.getByRole('button', { name: 'New collection', exact: true }).click();
|
||||
if (process.argv.includes('--expect-collections-bug')) {
|
||||
await page.waitForTimeout(200);
|
||||
assert.ok(errors.some(e => e.includes('randomUUID')), 'Expected to reproduce the creation failure');
|
||||
assert.equal(await page.getByLabel('Collection title', { exact: true }).count(), 0);
|
||||
console.log('REPRODUCED: New collection throws crypto.randomUUID is not a function on non-loopback HTTP');
|
||||
return;
|
||||
}
|
||||
await page.getByLabel('Collection title', { exact: true }).waitFor();
|
||||
await page.getByLabel('Collection title', { exact: true }).fill('Harbor Stories');
|
||||
await page.getByLabel('Collection description', { exact: true }).fill('A QA reading collection');
|
||||
assert.match(await status.textContent(), /Unsaved/);
|
||||
// An unrelated metadata action must not silently wipe a collection draft.
|
||||
await page.getByRole('button', { name: 'Import cached metadata suggestions', exact: true }).click();
|
||||
assert.match(await status.textContent(), /Save your changes/);
|
||||
assert.equal(await page.getByLabel('Collection title', { exact: true }).inputValue(), 'Harbor Stories');
|
||||
delaySave = true;
|
||||
const beforeSave = saveCount;
|
||||
await page.getByRole('button', { name: 'Save collection', exact: true }).click();
|
||||
assert.equal(await page.getByRole('button', { name: 'Save collection', exact: true }).isDisabled(), true);
|
||||
await page.evaluate(() => document.querySelector('fieldset button.btn.primary').click());
|
||||
await page.waitForFunction(() => document.querySelector('[role=status]')?.textContent === 'Saved');
|
||||
assert.equal(saveCount, beforeSave + 1, 'Duplicate save issued while one was in flight');
|
||||
delaySave = false;
|
||||
let state = await catalog();
|
||||
const created = state.collections.find(c => c.title === 'Harbor Stories');
|
||||
assert.ok(created); assert.match(created.id, /^[a-f0-9-]{36}$/);
|
||||
const harborId = state.mangas.find(m => m.title === 'Iron Harbor').id;
|
||||
const rainId = state.mangas.find(m => m.title === 'Before the Rain').id;
|
||||
const moonId = state.mangas.find(m => m.title === 'Moon Garden').id;
|
||||
assert.equal(created.members[0].mangaId, harborId);
|
||||
await page.reload();
|
||||
await page.getByLabel('Choose collection', { exact: true }).selectOption(created.id);
|
||||
assert.equal(await page.getByLabel('Collection title', { exact: true }).inputValue(), 'Harbor Stories');
|
||||
assert.equal(await page.getByLabel('Collection description', { exact: true }).inputValue(), 'A QA reading collection');
|
||||
await page.getByRole('button', { name: 'New collection', exact: true }).click();
|
||||
assert.match(await status.textContent(), /All manga are already grouped/);
|
||||
assert.equal(await page.locator('[data-member] button').isDisabled(), true, 'Last member must not be removable');
|
||||
// Server validation must be visible, recoverable, and retain the draft.
|
||||
await page.getByLabel('Collection title', { exact: true }).fill('');
|
||||
await page.getByRole('button', { name: 'Save collection', exact: true }).click();
|
||||
await page.getByText('Collection title is required', { exact: false }).waitFor();
|
||||
assert.equal(await page.getByRole('button', { name: 'Save collection', exact: true }).isEnabled(), true);
|
||||
await page.getByLabel('Collection title', { exact: true }).fill('Harbor Stories');
|
||||
await save();
|
||||
// Edit/reorder and remove a member from the original fixture collection.
|
||||
await page.getByLabel('Choose collection', { exact: true }).selectOption('moon-garden');
|
||||
const member = page.locator('[data-member]').filter({ hasText: 'Before the Rain' });
|
||||
await member.getByLabel('Member role').selectOption('side-story');
|
||||
await member.getByLabel('Reading order').fill('0');
|
||||
await page.getByLabel('Cover mode', { exact: true }).selectOption('selected');
|
||||
await page.getByLabel('Cover manga', { exact: true }).selectOption(rainId);
|
||||
await save();
|
||||
state = await catalog();
|
||||
assert.equal(state.collections.find(c => c.id === 'moon-garden').members[0].mangaId, rainId);
|
||||
assert.equal(state.collections.find(c => c.id === 'moon-garden').coverMangaId, rainId);
|
||||
await member.getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
await save();
|
||||
state = await catalog();
|
||||
assert.equal(state.collections.find(c => c.id === 'moon-garden').coverMangaId, moonId, 'Removing the cover member must repair the cover selection');
|
||||
await page.getByLabel('Choose collection', { exact: true }).selectOption(created.id);
|
||||
await page.getByLabel('Add manga to collection', { exact: true }).selectOption(rainId);
|
||||
await page.getByLabel('Cover mode', { exact: true }).selectOption('mosaic');
|
||||
await save();
|
||||
state = await catalog();
|
||||
assert.equal(state.collections.find(c => c.id === created.id).members.length, 2);
|
||||
assert.equal(new Set(state.collections.flatMap(c => c.members.map(m => m.mangaId))).size, 3);
|
||||
// Simulate a second admin's revision. A stale write must not overwrite it.
|
||||
const other = await fetch(url + '/api/admin/catalog', { method: 'POST', headers: { cookie, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ ...state, collections: state.collections.map(c => c.id === created.id ? { ...c, description: 'Changed by another editor' } : c) }) });
|
||||
assert.equal(other.status, 200);
|
||||
await page.getByLabel('Collection title', { exact: true }).fill('My unsaved title');
|
||||
await page.getByRole('button', { name: 'Save collection', exact: true }).click();
|
||||
await page.getByText('Catalog changed; reload before saving', { exact: false }).waitFor();
|
||||
assert.equal(await page.getByLabel('Collection title', { exact: true }).inputValue(), 'My unsaved title');
|
||||
page.once('dialog', d => d.dismiss());
|
||||
await page.getByRole('button', { name: 'Reload catalog', exact: true }).click();
|
||||
assert.equal(await page.getByLabel('Collection title', { exact: true }).inputValue(), 'My unsaved title');
|
||||
page.once('dialog', d => d.accept());
|
||||
await page.getByRole('button', { name: 'Reload catalog', exact: true }).click();
|
||||
await page.getByText('Catalog reloaded', { exact: true }).waitFor();
|
||||
assert.equal(await page.getByLabel('Collection description', { exact: true }).inputValue(), 'Changed by another editor');
|
||||
// Network/server failure is visible and retryable without losing edits.
|
||||
await page.getByLabel('Collection description', { exact: true }).fill('Retry kept my draft');
|
||||
failSave = true;
|
||||
await page.getByRole('button', { name: 'Save collection', exact: true }).click();
|
||||
await page.getByText('Fixture temporary save failure', { exact: false }).waitFor();
|
||||
assert.equal(await page.getByLabel('Collection description', { exact: true }).inputValue(), 'Retry kept my draft');
|
||||
await save('Save all changes');
|
||||
// Custom cover: real image encoding, backend validation, persisted selection.
|
||||
const png = new (require('./png').Canvas)(80, 120, [140, 60, 90]).png();
|
||||
await page.getByLabel('Custom cover', { exact: true }).setInputFiles({ name: 'qa.png', mimeType: 'image/png', buffer: png });
|
||||
await page.getByText('Cover saved', { exact: true }).waitFor();
|
||||
state = await catalog();
|
||||
const cover = state.collections.find(c => c.id === created.id).customCover;
|
||||
assert.ok(cover); assert.equal(state.collections.find(c => c.id === created.id).coverMode, 'custom');
|
||||
assert.equal(await page.getByLabel('Cover mode', { exact: true }).inputValue(), 'custom');
|
||||
assert.equal(await page.evaluate(id => fetch('/api/collection-cover?id=' + id).then(r => r.status), created.id), 200);
|
||||
await page.getByLabel('Custom cover', { exact: true }).setInputFiles({ name: 'large.png', mimeType: 'image/png', buffer: Buffer.alloc(600001) });
|
||||
await page.getByText('Choose an image under 600 KB.', { exact: true }).waitFor();
|
||||
assert.equal((await catalog()).collections.find(c => c.id === created.id).customCover, cover);
|
||||
// Relationships: create, reject, duplicate prevention, remove, cached import.
|
||||
await page.getByLabel('Relationship source', { exact: true }).selectOption(harborId);
|
||||
await page.getByLabel('Relationship target', { exact: true }).selectOption(moonId);
|
||||
await page.getByLabel('Relationship type', { exact: true }).selectOption('spin-off');
|
||||
await save('Add relationship');
|
||||
const relation = page.locator('[data-relation]').filter({ hasText: 'spin-off' });
|
||||
await relation.getByLabel('Relationship state', { exact: true }).selectOption('rejected');
|
||||
await page.waitForFunction(() => document.querySelector('[role=status]')?.textContent === 'Saved');
|
||||
assert.equal((await catalog()).relations.find(r => r.type === 'spin-off').state, 'rejected');
|
||||
await page.getByLabel('Relationship source', { exact: true }).selectOption(harborId);
|
||||
await page.getByLabel('Relationship target', { exact: true }).selectOption(moonId);
|
||||
await page.getByLabel('Relationship type', { exact: true }).selectOption('spin-off');
|
||||
const beforeDuplicate = saveCount;
|
||||
await page.getByRole('button', { name: 'Add relationship', exact: true }).click();
|
||||
assert.match(await status.textContent(), /already exists/); assert.equal(saveCount, beforeDuplicate);
|
||||
await relation.getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
await page.waitForFunction(() => document.querySelector('[role=status]')?.textContent === 'Saved');
|
||||
assert.ok(!(await catalog()).relations.some(r => r.type === 'spin-off'));
|
||||
await save('Import cached metadata suggestions');
|
||||
await page.getByRole('button', { name: 'Research relationships', exact: true }).click();
|
||||
await page.getByText('Assign a verified AniList identity first', { exact: true }).waitFor();
|
||||
await page.waitForFunction(() => !document.querySelector('fieldset').disabled);
|
||||
assert.equal(await page.getByRole('button', { name: 'Research relationships', exact: true }).isEnabled(), true);
|
||||
// Advanced JSON validation + round trip.
|
||||
await page.getByText('Advanced · JSON import/export & agent research packet', { exact: true }).click();
|
||||
await page.getByLabel('Catalog JSON', { exact: true }).fill('{');
|
||||
await page.getByRole('button', { name: 'Validate & save JSON', exact: true }).click();
|
||||
assert.equal(await page.getByLabel('Catalog JSON', { exact: true }).inputValue(), '{');
|
||||
await page.getByLabel('Choose collection', { exact: true }).selectOption('moon-garden');
|
||||
await page.getByLabel('Choose collection', { exact: true }).selectOption(created.id);
|
||||
await page.getByText('Advanced · JSON import/export & agent research packet', { exact: true }).click();
|
||||
assert.equal(await page.getByLabel('Catalog JSON', { exact: true }).inputValue(), '{', 'Changing collection discarded the JSON draft');
|
||||
const beforeJSON = saveCount;
|
||||
await page.getByRole('button', { name: 'Save all changes', exact: true }).click();
|
||||
assert.match(await status.textContent(), /Validate & save JSON first/);
|
||||
assert.equal(saveCount, beforeJSON, 'Ordinary save silently overwrote the JSON draft');
|
||||
await page.getByLabel('Catalog JSON', { exact: true }).fill(JSON.stringify(await catalog()));
|
||||
await save('Validate & save JSON');
|
||||
await noOverflow();
|
||||
await page.evaluate(() => scrollTo(0, 0));
|
||||
await page.screenshot({ path: path.join(shots, 'desktop-editor.png'), fullPage: true });
|
||||
// Mobile can edit + save too, not merely display the editor.
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await page.getByLabel('Collection description', { exact: true }).fill('Saved from mobile');
|
||||
await save(); await noOverflow();
|
||||
for (const row of await page.locator('[data-member]').all()) {
|
||||
const text = await row.locator('span').first().boundingBox();
|
||||
assert.ok(text.width > 150, 'Mobile member title is squeezed between controls');
|
||||
}
|
||||
await page.evaluate(() => scrollTo(0, 0));
|
||||
await page.screenshot({ path: path.join(shots, 'mobile-editor.png'), fullPage: true });
|
||||
// Reader presentation: grouped tiles, custom cover, order, expand/individual.
|
||||
await page.goto(origin + '/');
|
||||
await page.getByRole('link', { name: 'Harbor Stories', exact: false }).waitFor();
|
||||
assert.equal(await page.locator('.tile').count(), 2);
|
||||
await page.getByRole('link', { name: 'Harbor Stories', exact: false }).click();
|
||||
await page.getByRole('heading', { name: 'Harbor Stories', exact: true }).waitFor();
|
||||
assert.equal(await page.locator('.collection-member').count(), 2);
|
||||
assert.match(await page.locator('.collection-member').first().textContent(), /Iron Harbor/);
|
||||
assert.equal(await page.locator('.hero img').evaluate(img => img.complete && img.naturalWidth > 0), true);
|
||||
await page.goto(origin + '/');
|
||||
await page.getByLabel('Collection display', { exact: true }).selectOption('individual');
|
||||
assert.equal(await page.locator('.tile').count(), 3);
|
||||
await page.getByLabel('Collection display', { exact: true }).selectOption('expanded');
|
||||
assert.equal(await page.locator('.tile').count(), 3);
|
||||
assert.equal(await page.locator('.collection-heading').count(), 2);
|
||||
// Group flag and deletion never delete manga or reading content.
|
||||
await page.goto(origin + '/collections.html?id=' + created.id);
|
||||
await page.getByLabel('Group on the dashboard', { exact: true }).uncheck(); await save();
|
||||
assert.equal((await catalog()).collections.find(c => c.id === created.id).grouped, false);
|
||||
page.once('dialog', d => d.dismiss());
|
||||
await page.getByRole('button', { name: 'Delete collection', exact: true }).click();
|
||||
assert.ok((await catalog()).collections.some(c => c.id === created.id));
|
||||
page.once('dialog', d => d.accept());
|
||||
await save('Delete collection');
|
||||
state = await catalog(); assert.ok(!state.collections.some(c => c.id === created.id));
|
||||
assert.equal(state.mangas.length, 3); assert.equal(state.relations.length, 1);
|
||||
await page.reload();
|
||||
await page.getByLabel('Choose collection', { exact: true }).waitFor();
|
||||
assert.equal(await page.getByLabel('Choose collection', { exact: true }).locator('option').count(), 2);
|
||||
// A reader can see groups, but cannot retrieve/change the admin catalog.
|
||||
const result = await page.evaluate(async () => {
|
||||
const post = (p, b) => fetch(p, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(b) });
|
||||
const created = await post('/api/admin/user', { name: 'collections-reader-qa', password: 'disposable-reader-test', admin: false });
|
||||
if (!created.ok) return created.status;
|
||||
await post('/api/logout', {});
|
||||
return (await post('/api/login', { name: 'collections-reader-qa', password: 'disposable-reader-test' })).status;
|
||||
});
|
||||
assert.equal(result, 200);
|
||||
assert.equal(await page.evaluate(() => fetch('/api/admin/catalog').then(r => r.status)), 403);
|
||||
assert.equal(await page.evaluate(() => fetch('/api/admin/catalog', { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' }).then(r => r.status)), 403);
|
||||
await page.goto(origin + '/collections.html?qa=reader');
|
||||
await page.getByRole('link', { name: 'Back to the library', exact: true }).waitFor();
|
||||
assert.equal(await page.getByRole('button', { name: 'New collection', exact: true }).count(), 0);
|
||||
assert.equal(await page.evaluate(() => fetch('/api/collections').then(r => r.status)), 200);
|
||||
assert.deepEqual(errors, []);
|
||||
console.log('PASS: collections create/edit/save/reload, members/order/covers, relationships, JSON, failure/conflict recovery, mobile, dashboard and deletion on non-loopback HTTP');
|
||||
console.log('Screenshots: ' + shots);
|
||||
} catch (e) {
|
||||
await page.screenshot({ path: path.join(shots, 'failure.png'), fullPage: true }).catch(() => {});
|
||||
throw e;
|
||||
} finally { await browser.close(); }
|
||||
};
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// Isolated, repeatable container acceptance. Never mounts the real library/db/queue.
|
||||
const fs = require('fs'), os = require('os'), path = require('path'), crypto = require('crypto');
|
||||
const assert = require('assert/strict'), { spawn } = require('child_process');
|
||||
const { Canvas } = require('./png');
|
||||
const root = path.resolve(__dirname, '..'), dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-compose-'));
|
||||
const project = 'mc-qa-' + crypto.randomBytes(3).toString('hex'), port = process.env.QA_PORT || '8180';
|
||||
const library = path.join(dir, 'library'), data = path.join(dir, 'data'), tokenFile = path.join(dir, 'token');
|
||||
const directionQa = process.argv.includes('--direction-ui');
|
||||
fs.mkdirSync(path.join(data, 'scraper'), { recursive: true }); fs.mkdirSync(path.join(data, 'sentinel'));
|
||||
fs.writeFileSync(path.join(data, 'scraper', 'queue.json'), JSON.stringify({ paused: true, queue: [], settings: { autoUpdateEnabled: false } }));
|
||||
fs.writeFileSync(tokenFile, crypto.randomBytes(32).toString('hex'), { mode: 0o600 });
|
||||
const png = new Canvas(180, 270, [60, 90, 140]).png();
|
||||
for (const [index, name] of ['Moon Garden', 'Before the Rain', 'Iron Harbor'].entries()) {
|
||||
const folder = path.join(library, name, 'v01', 'c0001 The Beginning'); fs.mkdirSync(folder, { recursive: true });
|
||||
fs.writeFileSync(path.join(folder, '0001.png'), png); fs.writeFileSync(path.join(library, name, 'cover.png'), png);
|
||||
if (process.argv.includes('--reading-ui') && index === 0) {
|
||||
for (let page = 2; page <= 20; page++) fs.writeFileSync(path.join(folder, String(page).padStart(4, '0') + '.png'), png);
|
||||
}
|
||||
if (directionQa && index === 0) {
|
||||
fs.writeFileSync(path.join(folder, '0001.png'), new Canvas(270, 180, [60, 90, 140]).png());
|
||||
for (let page = 2; page <= 4; page++) fs.writeFileSync(path.join(folder, String(page).padStart(4, '0') + '.png'), new Canvas(180, 1000, [60, 90, 140]).png());
|
||||
}
|
||||
fs.writeFileSync(path.join(library, name, 'manga.json'), JSON.stringify({ title: name,
|
||||
direction: directionQa && index === 1 ? 'ltr' : process.argv.includes('--reading-ui') && index === 0 ? 'ttb' : 'rtl',
|
||||
...(directionQa && index === 0 ? { 'x-reading-direction': { version: 1, value: 'rtl', source: 'fallback' } } : {}),
|
||||
status: index ? 'completed' : 'ongoing', tags: ['Fantasy', 'Demo'] }));
|
||||
}
|
||||
const env = { ...process.env, LIBRARY_PATH: library, DATA_PATH: data, SCRAPER_DATA_PATH: path.join(data, 'scraper'), SENTINEL_DATA_PATH: path.join(data, 'sentinel'),
|
||||
SCRAPER_TOKEN_PATH: tokenFile, READER_PORT: port, READER_BIND: '127.0.0.1', COOKIE_SECURE: '0', APP_IMAGE: process.env.APP_IMAGE || 'mangacircus:local' };
|
||||
const args = ['compose', '-p', project, '-f', path.join(root, 'docker-compose.yml')];
|
||||
const docker = (...command) => new Promise((resolve, reject) => {
|
||||
const child = spawn('docker', [...args, ...command], { cwd: root, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let result = ''; child.stdout.on('data', chunk => result += chunk); child.stderr.on('data', chunk => process.stderr.write(chunk));
|
||||
child.on('error', reject); child.once('exit', code => code ? reject(new Error(`Docker ${command[0]} failed (${code})`)) : resolve(result));
|
||||
});
|
||||
const pause = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
async function until(check, message, attempts = 100) {
|
||||
for (let n = 0; n < attempts; n++) { try { if (await check()) return; } catch {} await pause(300); }
|
||||
throw new Error(message);
|
||||
}
|
||||
let keep = false;
|
||||
(async () => {
|
||||
try {
|
||||
await docker('config', '-q'); await docker('up', '-d', '--no-build');
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
await until(() => fetch(url + '/health').then(r => r.ok), 'Reader did not start');
|
||||
const signup = await fetch(url + '/api/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'qa', password: 'disposable-local-test' }) });
|
||||
assert.equal(signup.status, 200); const cookie = signup.headers.get('set-cookie').split(';')[0];
|
||||
const api = (endpoint, body) => fetch(url + endpoint, { headers: { cookie, 'content-type': 'application/json' }, ...(body ? { method: 'POST', body: JSON.stringify(body) } : {}) });
|
||||
let inventory;
|
||||
await until(async () => { inventory = await (await api('/api/library')).json(); return inventory.mangas?.length === 3; }, 'Fixture inventory missing');
|
||||
const byTitle = title => inventory.mangas.find(m => m.title === title).id;
|
||||
const catalog = { revision: 0, collections: [{ id: 'moon-garden', title: 'Moon Garden Universe', coverMode: 'mosaic', members: [
|
||||
{ mangaId: byTitle('Moon Garden'), role: 'main', readingOrder: 1 }, { mangaId: byTitle('Before the Rain'), role: 'prequel', readingOrder: 2 }] }],
|
||||
relations: [{ from: byTitle('Moon Garden'), to: byTitle('Before the Rain'), type: 'prequel', state: 'confirmed' }] };
|
||||
assert.equal((await api('/api/admin/catalog', catalog)).status, 200);
|
||||
assert.equal((await api('/api/collection?id=moon-garden')).status, 200);
|
||||
let state = await (await api('/api/admin/scraper')).json(); assert.equal(state.available, true); assert.equal(state.paused, true); assert.equal(state.version, 4);
|
||||
const cleanup = await api('/api/admin/scraper', { command: 'clear-completed' });
|
||||
assert.equal(cleanup.status, 200);
|
||||
assert.equal((await cleanup.json()).cleared, 0);
|
||||
assert.ok(state.libraryStatus);
|
||||
await until(async () => { state = await (await api('/api/admin/scraper')).json(); return state.libraryStatus?.ready && state.library.length === 3; }, 'Background scraper inventory missing');
|
||||
const settingsJs = await (await fetch(url + '/settings.js')).text();
|
||||
assert.ok(!settingsJs.includes('Start it with: npm run start:scraper'));
|
||||
await until(async () => { const h = await (await api('/api/admin/health')).json(); return h.sentinel.services?.reader && h.sentinel.services?.scraper; }, 'Internal Sentinel networking failed');
|
||||
const mounts = JSON.parse(await docker('config', '--format', 'json'));
|
||||
assert.ok(mounts.services.reader.volumes.find(v => v.target === '/library').read_only);
|
||||
assert.ok(!mounts.services.scraper.ports?.length && !mounts.services.sentinel.ports?.length);
|
||||
const denied = JSON.parse(await docker('exec', '-T', 'reader', 'node', '-e', "Promise.all(['http://scraper:8091/api/state','http://sentinel:8092/api/state'].map(u=>fetch(u).then(r=>r.status))).then(v=>console.log(JSON.stringify(v)))"));
|
||||
assert.deepEqual(denied, [401, 401]);
|
||||
// Deliberate fixture-only child crash: its supervisor must recover just the reader.
|
||||
const before = JSON.parse(fs.readFileSync(path.join(data, 'runtime-reader.json'))).pid;
|
||||
await docker('exec', '-T', 'reader', 'node', '-e', "const fs=require('fs');process.kill(JSON.parse(fs.readFileSync('/data/runtime-reader.json')).pid,'SIGKILL')");
|
||||
await until(async () => { const runtime = JSON.parse(fs.readFileSync(path.join(data, 'runtime-reader.json'))); return runtime.pid !== before && runtime.status === 'running' && (await fetch(url + '/health')).ok; }, 'Reader child did not recover');
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(data, 'scraper', 'runtime.json'))).failures.length, 0);
|
||||
const image = await api('/lib/' + encodeURIComponent('Moon Garden') + '/cover.png'); assert.equal(image.status, 200);
|
||||
state = await (await api('/api/admin/scraper')).json(); assert.equal(state.paused, true);
|
||||
if (process.argv.includes('--settings-ui')) await require('./test-settings-ui')(url, cookie);
|
||||
if (process.argv.includes('--collections-ui')) {
|
||||
// Each UI suite tests logout/ordinary-reader boundaries. Give the next suite
|
||||
// a fresh disposable admin session rather than reusing a logged-out cookie.
|
||||
const login = await api('/api/login', { name: 'qa', password: 'disposable-local-test' });
|
||||
assert.equal(login.status, 200);
|
||||
await require('./test-collections-ui')(url, login.headers.get('set-cookie').split(';')[0]);
|
||||
}
|
||||
if (process.argv.includes('--queue-ui')) {
|
||||
const login = await api('/api/login', { name: 'qa', password: 'disposable-local-test' });
|
||||
assert.equal(login.status, 200);
|
||||
await require('./test-queue-ui')(url, login.headers.get('set-cookie').split(';')[0]);
|
||||
}
|
||||
if (process.argv.includes('--reading-ui')) {
|
||||
const login = await api('/api/login', { name: 'qa', password: 'disposable-local-test' });
|
||||
assert.equal(login.status, 200);
|
||||
await require('./test-reading-ui')(url, login.headers.get('set-cookie').split(';')[0]);
|
||||
}
|
||||
if (directionQa) {
|
||||
const login = await api('/api/login', { name: 'qa', password: 'disposable-local-test' });
|
||||
assert.equal(login.status, 200);
|
||||
await require('./test-direction-ui')(url, login.headers.get('set-cookie').split(';')[0]);
|
||||
}
|
||||
console.log('PASS: isolated Compose auth, collections, read-only reader, private APIs, Sentinel networking and child crash recovery');
|
||||
keep = process.argv.includes('--keep');
|
||||
if (keep) {
|
||||
const envFile = path.join(dir, 'qa.env');
|
||||
fs.writeFileSync(envFile, ['LIBRARY_PATH', 'DATA_PATH', 'SCRAPER_DATA_PATH', 'SENTINEL_DATA_PATH', 'SCRAPER_TOKEN_PATH', 'READER_PORT', 'READER_BIND', 'COOKIE_SECURE', 'APP_IMAGE'].map(k => `${k}=${env[k].replaceAll('\\', '/')}`).join('\n') + '\n');
|
||||
console.log(JSON.stringify({ project, url, envFile, fixture: dir }));
|
||||
}
|
||||
} finally {
|
||||
if (!keep) { await docker('down').catch(() => {}); fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
})().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
@@ -0,0 +1,34 @@
|
||||
'use strict';
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
module.exports = async function (url, cookie) {
|
||||
const { chromium } = require('playwright');
|
||||
const browser = await chromium.launch({ channel: process.env.QA_BROWSER_CHANNEL || 'msedge', headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true });
|
||||
const [name, ...value] = cookie.split('=');
|
||||
await context.addCookies([{ name, value: value.join('='), url }]);
|
||||
try {
|
||||
const api = async path => {
|
||||
const response = await context.request.get(url + path);
|
||||
assert.equal(response.ok(), true, await response.text()); return response.json();
|
||||
};
|
||||
const inventory = await api('/api/library');
|
||||
const pick = async title => {
|
||||
const manga = inventory.mangas.find(item => item.title === title);
|
||||
const detail = await api('/api/manga?m=' + encodeURIComponent(manga.id));
|
||||
const chapter = detail.chapters[0];
|
||||
return { manga, chapter, data: await api('/api/chapter?m=' + encodeURIComponent(manga.id) + '&c=' + encodeURIComponent(chapter.id)) };
|
||||
};
|
||||
const strip = await pick('Moon Garden');
|
||||
const printed = await pick('Before the Rain');
|
||||
assert.equal(strip.data.direction, 'ttb');
|
||||
assert.equal(printed.data.direction, 'ltr');
|
||||
const page = await context.newPage();
|
||||
await page.goto(url + '/read.html?m=' + encodeURIComponent(strip.manga.id) + '&c=' + encodeURIComponent(strip.chapter.id));
|
||||
await page.waitForFunction(() => document.querySelectorAll('#canvas img').length === 4);
|
||||
const widths = await page.locator('#canvas img').evaluateAll(images => images.map(img => Math.round(img.getBoundingClientRect().width)));
|
||||
assert.equal(widths.length, 4);
|
||||
assert.ok(widths.every(width => Math.abs(width - widths[1]) <= 1), `Flow widths differ: ${widths.join(', ')}`);
|
||||
console.log('PASS: mobile flow gives the landscape credits card the same width as detected webtoon strips; printed manual LTR remains LTR');
|
||||
} finally { await context.close(); await browser.close(); }
|
||||
};
|
||||
@@ -0,0 +1,101 @@
|
||||
'use strict';
|
||||
// Fixture-only state interception: no source traffic or production accounts.
|
||||
const assert = require('node:assert/strict'), fs = require('node:fs'), path = require('node:path');
|
||||
module.exports = async function testQueueUI(url, cookie) {
|
||||
const { chromium } = require('playwright');
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 1280, height: 900 } });
|
||||
const [name, ...values] = cookie.split('=');
|
||||
await context.addCookies([{ name, value: values.join('='), url }]);
|
||||
const page = await context.newPage(), errors = [], commands = [], upstream = [];
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
page.on('request', request => { if (new URL(request.url()).origin !== new URL(url).origin) upstream.push(request.url()); });
|
||||
const state = await (await fetch(url + '/api/admin/scraper', { headers: { cookie } })).json();
|
||||
state.library[0].sources = [{ name: 'MangaDex', adapter: 'mangadex', url: 'https://mangadex.org/title/fixture' }];
|
||||
state.updateChecks = [{ mangaId: state.library[0].id, source: 'mangadex', lastCheckedAt: '2026-10-07T08:00:00Z', lastDownloadedAt: '2026-10-07T07:00:00Z', lastResult: 'no-updates' }];
|
||||
state.history = [
|
||||
{ title: 'Moon Garden', source: 'mangadex', status: 'checked', time: '2026-10-07T08:00:00Z' },
|
||||
{ title: 'Another manga', source: 'weebcentral', status: 'checked', time: '2026-10-07T07:00:00Z' },
|
||||
];
|
||||
state.queue = [
|
||||
{ id: 'pilot-fixture', title: 'bleach', source: 'mangadex', kind: 'inspection', status: 'ready', sourceTitle: 'Bleach (Pre-Serialization)', availability: { selectedChapters: 2, specialReleases: 2, coverage: 'unknown' } },
|
||||
{ id: 'main-fixture', title: 'Bleach', source: 'mangadex', kind: 'download', status: 'complete', publishedCount: 1, sourceTitle: 'Bleach', availability: { selectedChapters: 1, expectedLastChapter: '686', coverage: 'partial' } },
|
||||
{ id: 'external-fixture', title: 'Claymore', source: 'mangadex', kind: 'download', status: 'unavailable', error: 'MangaDex lists 8 English external-link releases, but no hosted English pages. External links are not downloaded.' },
|
||||
{ id: 'other-source-fixture', title: 'Another manga', source: 'weebcentral', kind: 'download', status: 'queued' },
|
||||
];
|
||||
await page.route('**/api/admin/scraper', async route => {
|
||||
if (route.request().method() === 'POST') {
|
||||
const command = route.request().postDataJSON(); commands.push(command);
|
||||
if (command.action === 'confirm') state.queue.find(item => item.id === command.id).status = 'queued';
|
||||
if (command.action === 'inspect') state.queue.find(item => item.id === command.id).status = 'queued';
|
||||
if (command.command === 'clear-completed') state.queue = state.queue.filter(item => item.status !== 'complete' || item.kind !== 'download');
|
||||
}
|
||||
await route.fulfill({ contentType: 'application/json', body: JSON.stringify(state) });
|
||||
});
|
||||
const shots = path.resolve(__dirname, '..', 'data', 'qa', 'queue'); fs.mkdirSync(shots, { recursive: true });
|
||||
try {
|
||||
await page.goto(url + '/settings.html#admin/scraper');
|
||||
const activity = page.locator('[data-scraper-tab=activity]'), queuePanel = page.locator('[data-scraper-tab=queue]');
|
||||
await activity.getByText('Source edition: Bleach (Pre-Serialization)', { exact: true }).waitFor();
|
||||
await activity.getByText(/no hosted English pages/).waitFor();
|
||||
assert.equal(commands.length, 0, 'Inspection status must not start a download');
|
||||
await page.screenshot({ path: path.join(shots, 'desktop.png'), fullPage: true });
|
||||
await activity.getByRole('button', { name: 'Confirm download', exact: true }).click();
|
||||
await page.waitForFunction(() => ![...document.querySelectorAll('button')].some(b => b.textContent === 'Confirm download'));
|
||||
assert.deepEqual(commands.at(-1), { command: 'control', action: 'confirm', id: 'pilot-fixture' });
|
||||
await activity.locator('.scraper-item').filter({ hasText: 'Claymore' }).getByRole('button', { name: 'Inspect again', exact: true }).click();
|
||||
assert.deepEqual(commands.at(-1), { command: 'control', action: 'inspect', id: 'external-fixture' });
|
||||
await page.getByRole('link', { name: 'Queue', exact: true }).click();
|
||||
await queuePanel.getByText('Completed downloads · 1').click();
|
||||
await page.waitForTimeout(3500);
|
||||
assert.equal(await queuePanel.locator('.scraper-completed').evaluate(el => el.open), true, 'Completed section collapsed on polling');
|
||||
// Rebuild the panel during the summary click, before the native toggle.
|
||||
// A state snapshot taken only at redraw time misses this click and recloses it.
|
||||
await queuePanel.getByText('Completed downloads · 1').click();
|
||||
await page.evaluate(() => {
|
||||
const summary = document.querySelector('.scraper-completed > summary');
|
||||
summary.addEventListener('click', () => {
|
||||
const buttons = document.querySelectorAll('.scraper-filters button');
|
||||
buttons[1].click(); buttons[0].click();
|
||||
}, { capture: true, once: true });
|
||||
});
|
||||
await queuePanel.getByText('Completed downloads · 1').click();
|
||||
await page.waitForTimeout(3500);
|
||||
assert.equal(await queuePanel.locator('.scraper-completed').evaluate(el => el.open), true, 'Completed section lost a click during redraw');
|
||||
await queuePanel.getByText(/1 chapter published · finished/).waitFor();
|
||||
await queuePanel.getByText(/Partial source coverage/).waitFor();
|
||||
page.once('dialog', dialog => dialog.accept());
|
||||
await page.getByRole('button', { name: 'Clear completed downloads (1)' }).click();
|
||||
assert.deepEqual(commands.at(-1), { command: 'clear-completed', source: '' });
|
||||
assert.equal(await queuePanel.locator('.scraper-completed').evaluate(el => el.open), true, 'Completed section collapsed after cleanup');
|
||||
assert.equal(state.queue.some(item => item.id === 'main-fixture'), false);
|
||||
assert.ok(state.queue.some(item => item.id === 'pilot-fixture'));
|
||||
await page.getByRole('button', { name: 'MangaDex', exact: true }).click();
|
||||
assert.equal(await page.getByText('Another manga').count(), 0);
|
||||
await page.getByRole('button', { name: 'All sources', exact: true }).click();
|
||||
await queuePanel.getByText('Another manga').waitFor();
|
||||
await page.getByRole('link', { name: 'Updates', exact: true }).click();
|
||||
await page.getByText(/Last checked:.*2026/).waitFor();
|
||||
await page.getByText(/Last downloaded:.*2026/).waitFor();
|
||||
await page.getByRole('button', { name: 'MangaDex', exact: true }).click();
|
||||
await page.getByText('Recent check activity').click();
|
||||
await page.waitForTimeout(3500);
|
||||
assert.equal(await page.locator('.scraper-history').evaluate(el => el.open), true, 'Check history collapsed on polling');
|
||||
assert.equal(await page.locator('[data-scraper-tab=updates] .scraper-history').getByText('Another manga', { exact: false }).count(), 0);
|
||||
await page.getByRole('button', { name: 'All sources', exact: true }).click();
|
||||
await page.getByRole('link', { name: 'Queue', exact: true }).click();
|
||||
await page.getByText('Add manga to queue').click();
|
||||
await page.waitForTimeout(3500);
|
||||
assert.equal(await queuePanel.locator('.scraper-add').evaluate(el => el.open), true, 'Add form collapsed on polling');
|
||||
await page.locator('input[type=url]').fill('https://mangadex.org/title/06a37094-b3ba-4f25-bf2b-54c9751e0e91');
|
||||
await page.getByRole('button', { name: 'Inspect & queue', exact: true }).click();
|
||||
assert.equal(commands.at(-1).inspect, true); assert.equal(commands.at(-1).command, 'enqueue');
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth));
|
||||
await page.screenshot({ path: path.join(shots, 'mobile.png'), fullPage: true });
|
||||
const detailWidth = await page.locator('[data-scraper-tab=queue] .scraper-queue .scraper-item > .grow').first().evaluate(el => el.getBoundingClientRect().width);
|
||||
assert.ok(detailWidth > 280, `Queue details squeezed by action buttons (${detailWidth}px)`);
|
||||
assert.deepEqual(errors, []); assert.deepEqual(upstream, []);
|
||||
console.log('PASS: scraper activity, queue cleanup, source filters, edition controls, add form and mobile layout; no source traffic');
|
||||
} finally { await browser.close(); }
|
||||
};
|
||||
@@ -0,0 +1,87 @@
|
||||
'use strict';
|
||||
// Browser QA uses ONLY test-compose's disposable user/library, never live history.
|
||||
const assert = require('node:assert/strict');
|
||||
module.exports = async function (url, cookie) {
|
||||
const { chromium } = require('playwright');
|
||||
const browser = await chromium.launch({ channel: process.env.QA_BROWSER_CHANNEL || 'msedge', headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true });
|
||||
const [name, ...value] = cookie.split('=');
|
||||
await context.addCookies([{ name, value: value.join('='), url }]);
|
||||
const page = await context.newPage(), errors = [], packets = [];
|
||||
page.on('pageerror', e => errors.push(e.message));
|
||||
let delayed = false;
|
||||
context.on('request', request => {
|
||||
if (!request.url().endsWith('/api/reading/visit') || request.method() !== 'POST') return;
|
||||
try { packets.push(request.postDataJSON()); } catch {}
|
||||
});
|
||||
await context.route(url + '/api/reading/visit', async route => {
|
||||
const body = route.request().postDataJSON();
|
||||
if (delayed && !body.final && body.activeMs > 0) { delayed = false; await new Promise(resolve => setTimeout(resolve, 1000)); }
|
||||
const response = await route.fetch(); await route.fulfill({ response });
|
||||
});
|
||||
const api = async (path, body) => {
|
||||
const response = await context.request.fetch(url + path, { method: body ? 'POST' : 'GET', data: body });
|
||||
assert.equal(response.ok(), true, await response.text()); return response.json();
|
||||
};
|
||||
try {
|
||||
await api('/api/settings', { readingClock: true, startMode: 'flow', flowPageGap: false });
|
||||
const inventory = await api('/api/library'), manga = inventory.mangas.find(m => m.title === 'Moon Garden');
|
||||
const detail = await api('/api/manga?m=' + encodeURIComponent(manga.id));
|
||||
const chapter = detail.chapters[0];
|
||||
const reader = url + '/read.html?m=' + encodeURIComponent(manga.id) + '&c=' + encodeURIComponent(chapter.id);
|
||||
const summary = async () => (await api('/api/chapter?m=' + encodeURIComponent(manga.id) + '&c=' + encodeURIComponent(chapter.id))).reading;
|
||||
const load = async () => {
|
||||
await page.goto(reader); await page.locator('#readclock:not([hidden])').waitFor();
|
||||
await page.waitForFunction(() => [...document.querySelectorAll('#canvas img')].some(img => img.complete && img.naturalWidth));
|
||||
await page.locator('#stage').focus();
|
||||
// Loaded pages, fast actual observed positions; no fictional intermediate views.
|
||||
await page.waitForTimeout(130);
|
||||
};
|
||||
await load();
|
||||
const images = page.locator('#canvas img');
|
||||
for (let i = 0; i < 12; i++) {
|
||||
await images.nth(i).evaluate(img => { const stage = document.querySelector('#stage'); stage.scrollTop += img.getBoundingClientRect().top - stage.getBoundingClientRect().top; });
|
||||
await page.waitForFunction(index => { const img = document.querySelectorAll('#canvas img')[index]; return img?.complete && img.naturalWidth; }, i);
|
||||
await page.waitForTimeout(125);
|
||||
}
|
||||
// Flow auto-hides the bars while scrolling; inspect the clock panel without
|
||||
// introducing another page-navigation gesture into the coverage test.
|
||||
await page.locator('#readclock').dispatchEvent('click');
|
||||
await page.getByText('Yes · time continues counting', { exact: true }).waitFor();
|
||||
// Force a normal in-flight save, then exercise pagehide explicitly before
|
||||
// Playwright's programmatic navigation tears down the old document.
|
||||
// The final beacon must not wait for the delayed ordinary save.
|
||||
delayed = true;
|
||||
await page.evaluate(() => window.dispatchEvent(new Event('blur')));
|
||||
await page.evaluate(() => window.dispatchEvent(new Event('pagehide')));
|
||||
await page.waitForTimeout(100);
|
||||
await page.goto(url + '/manga.html?m=' + encodeURIComponent(manga.id));
|
||||
await page.waitForTimeout(1400);
|
||||
const saved = await summary();
|
||||
assert.equal(saved.completedPasses, 1); assert.ok(saved.totalActiveMs >= 1000);
|
||||
assert.ok(packets.some(p => p.final && p.seen.length >= 12),
|
||||
`Final coverage packet missing (closed=${saved.currentPass?.closed}): ${JSON.stringify(packets.map(p => ({ seq: p.seq, final: p.final, seen: p.seen?.length, activeMs: p.activeMs })))}`);
|
||||
const before = saved.totalActiveMs;
|
||||
await load();
|
||||
// Jump to one picture and dwell. Prior read must not lend this visit coverage.
|
||||
await images.nth(18).evaluate(img => { const stage = document.querySelector('#stage'); stage.scrollTop += img.getBoundingClientRect().top - stage.getBoundingClientRect().top; });
|
||||
await page.waitForTimeout(1500);
|
||||
await page.goto(url + '/manga.html?m=' + encodeURIComponent(manga.id));
|
||||
await page.waitForTimeout(400);
|
||||
const lookup = await summary();
|
||||
assert.equal(lookup.totalActiveMs, before); assert.equal(lookup.completedPasses, 1);
|
||||
assert.ok(lookup.currentPass.seenPages < 12);
|
||||
// A stationary reader must pulse periodically, not continuously after replies.
|
||||
await load(); const count = packets.length;
|
||||
await page.waitForTimeout(11500);
|
||||
assert.ok(packets.length - count <= 2, 'Response-driven request storm');
|
||||
await page.waitForFunction(() => document.querySelector('#readclock .clock-label')?.textContent.includes('Idle'), null, { timeout: 23000 });
|
||||
const pausedTime = await page.locator('#readclock .clock-time').textContent();
|
||||
await page.waitForTimeout(1100);
|
||||
assert.equal(await page.locator('#readclock .clock-time').textContent(), pausedTime, 'Idle time still accrued');
|
||||
await page.mouse.move(20, 200);
|
||||
await page.waitForFunction(() => document.querySelector('#readclock .clock-label')?.textContent.includes('Reading'));
|
||||
assert.equal(errors.length, 0, errors.join('\n'));
|
||||
console.log('PASS: mobile flow reader fast coverage, 60% exit save with pulse in flight, lookup exclusion, bounded pulse frequency, 30-second idle pause and input resume');
|
||||
} finally { await context.close(); await browser.close(); }
|
||||
};
|
||||
@@ -0,0 +1,109 @@
|
||||
'use strict';
|
||||
// Optional headless browser QA against the disposable Compose fixture only.
|
||||
// Playwright is a development tool, not an application dependency.
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs'), path = require('node:path');
|
||||
|
||||
module.exports = async function testSettingsUI(url, cookie) {
|
||||
const { chromium } = require('playwright');
|
||||
const browser = await chromium.launch({ channel: process.env.QA_BROWSER_CHANNEL || 'msedge', headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 1280, height: 900 } });
|
||||
const [name, ...value] = cookie.split('=');
|
||||
await context.addCookies([{ name, value: value.join('='), url }]);
|
||||
const page = await context.newPage(), errors = [], requests = [];
|
||||
page.on('pageerror', e => errors.push(e.message));
|
||||
page.on('request', r => requests.push(new URL(r.url()).pathname));
|
||||
const shots = path.resolve(__dirname, '..', 'data', 'qa', 'settings');
|
||||
fs.mkdirSync(shots, { recursive: true });
|
||||
const clickSection = async (id, title) => {
|
||||
await page.locator(`.settings-menu a[href="settings.html#${id}"]`).click();
|
||||
await page.getByRole('heading', { name: title, exact: true, level: 1 }).waitFor();
|
||||
};
|
||||
const noOverflow = async () => assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), 'Horizontal page overflow');
|
||||
try {
|
||||
await page.goto(url + '/settings.html');
|
||||
await page.getByRole('heading', { name: 'Reading', exact: true, level: 1 }).waitFor();
|
||||
assert.equal(await page.locator('.settings-menu-group').count(), 2);
|
||||
assert.equal(requests.filter(p => p.startsWith('/api/admin')).length, 0, 'Personal settings fetched admin data');
|
||||
await page.getByLabel('Skip chapter cover pages', { exact: true }).check();
|
||||
await page.waitForFunction(async () => (await (await fetch('/api/me')).json()).settings.skipCovers === true);
|
||||
await page.screenshot({ path: path.join(shots, 'desktop-reading.png'), fullPage: true });
|
||||
await clickSection('user/notifications', 'Notifications');
|
||||
const webhook = page.locator('input[type=url]');
|
||||
await webhook.fill('https://example.test/unsaved-draft');
|
||||
await clickSection('admin/scraper', 'Scraper');
|
||||
await page.getByRole('heading', { name: 'Activity', exact: false, level: 2 }).waitFor();
|
||||
assert.equal(await page.locator('[data-scraper-tab]:visible').getAttribute('data-scraper-tab'), 'activity');
|
||||
await page.getByRole('link', { name: 'Sources', exact: true }).click();
|
||||
const min = page.locator('[data-scraper-tab=sources] input[type=number]').first();
|
||||
await min.fill('1.5');
|
||||
await page.getByRole('link', { name: 'Updates', exact: true }).click();
|
||||
await page.getByRole('link', { name: 'Sources', exact: true }).click();
|
||||
assert.equal(await min.inputValue(), '1.5', 'Tab switch discarded a draft');
|
||||
await page.waitForTimeout(3300);
|
||||
assert.equal(await min.inputValue(), '1.5', 'Polling discarded a draft');
|
||||
await page.getByRole('button', { name: 'Save WeebCentral', exact: true }).click();
|
||||
await page.waitForFunction(() => document.querySelector('[data-scraper-tab=sources] input[type=number]')?.value === '1.5');
|
||||
const state = await page.evaluate(() => fetch('/api/admin/scraper').then(r => r.json()));
|
||||
assert.equal(state.settings.sourceRates.weebcentral.minMs, 1500);
|
||||
await noOverflow();
|
||||
await page.screenshot({ path: path.join(shots, 'desktop-hosters.png'), fullPage: true });
|
||||
await clickSection('user/notifications', 'Notifications');
|
||||
assert.equal(await page.locator('input[type=url]').inputValue(), 'https://example.test/unsaved-draft');
|
||||
await page.waitForTimeout(300);
|
||||
const before = requests.filter(p => p === '/api/admin/scraper').length;
|
||||
await page.waitForTimeout(3300);
|
||||
assert.equal(requests.filter(p => p === '/api/admin/scraper').length, before, 'Hidden scraper keeps polling');
|
||||
await page.goBack();
|
||||
await page.locator('[data-scraper-tab=sources]:visible').waitFor();
|
||||
await page.goForward();
|
||||
await page.getByRole('heading', { name: 'Notifications', exact: true, level: 1 }).waitFor();
|
||||
await clickSection('user/reading-time', 'Reading time & privacy');
|
||||
await page.getByText('Track and show the Active Reading Clock').waitFor();
|
||||
await clickSection('admin/library', 'Server & library');
|
||||
await page.getByRole('button', { name: 'Full rescan', exact: true }).waitFor();
|
||||
assert.equal(await page.getByRole('button', { name: 'Save user', exact: true }).count(), 0);
|
||||
await clickSection('admin/users', 'Users & access');
|
||||
await page.getByRole('button', { name: 'Save user', exact: true }).waitFor();
|
||||
assert.equal(await page.getByRole('button', { name: 'Full rescan', exact: true }).count(), 0);
|
||||
await page.locator('.settings-menu a[href="collections.html"]').click();
|
||||
await page.getByRole('heading', { name: 'Collections & metadata', exact: true, level: 1 }).waitFor();
|
||||
assert.equal(await page.locator('.settings-menu a[aria-current=page]').getAttribute('href'), 'collections.html');
|
||||
await page.locator('.settings-menu a[href="health.html"]').click();
|
||||
await page.getByRole('heading', { name: 'Service health', exact: true, level: 1 }).waitFor();
|
||||
await page.getByText('No failed jobs.', { exact: true }).waitFor();
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await noOverflow();
|
||||
await clickSection('admin/scraper', 'Scraper');
|
||||
await page.getByRole('heading', { name: 'Activity', exact: false, level: 2 }).waitFor();
|
||||
await noOverflow();
|
||||
await page.screenshot({ path: path.join(shots, 'mobile-queue.png'), fullPage: true });
|
||||
await page.getByRole('link', { name: 'Sources', exact: true }).click();
|
||||
await noOverflow();
|
||||
await page.screenshot({ path: path.join(shots, 'mobile-hosters.png'), fullPage: true });
|
||||
// Create/login a fixture-only ordinary reader. No production accounts are touched.
|
||||
const result = await page.evaluate(async () => {
|
||||
const post = (p, b) => fetch(p, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(b) });
|
||||
const created = await post('/api/admin/user', { name: 'ordinary-qa', password: 'disposable-reader-test', admin: false });
|
||||
if (!created.ok) return created.status;
|
||||
await post('/api/logout', {});
|
||||
return (await post('/api/login', { name: 'ordinary-qa', password: 'disposable-reader-test' })).status;
|
||||
});
|
||||
assert.equal(result, 200);
|
||||
const readerRequestStart = requests.length;
|
||||
// Change the document URL, not just its hash, after switching fixture accounts.
|
||||
await page.goto(url + '/settings.html?qa=ordinary#admin/users');
|
||||
await page.getByRole('heading', { name: 'Reading', exact: true, level: 1 }).waitFor();
|
||||
assert.equal(await page.locator('.settings-menu-group').count(), 1);
|
||||
assert.equal(await page.locator('.settings-menu a[href*="admin"]').count(), 0);
|
||||
assert.equal(requests.slice(readerRequestStart).filter(p => p.startsWith('/api/admin')).length, 0, 'Ordinary reader fetched admin data');
|
||||
assert.equal(await page.evaluate(() => fetch('/api/admin').then(r => r.status)), 403);
|
||||
await noOverflow();
|
||||
assert.deepEqual(errors, []);
|
||||
console.log('PASS: settings desktop/mobile navigation, deep links, role boundaries, draft retention, saves and polling lifecycle');
|
||||
console.log('Screenshots: ' + shots);
|
||||
} catch (e) {
|
||||
await page.screenshot({ path: path.join(shots, 'failure.png'), fullPage: true }).catch(() => {});
|
||||
throw e;
|
||||
} finally { await browser.close(); }
|
||||
};
|
||||
Reference in new issue
Block a user