Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e3e57b60bd | ||
|
|
03767d5704 | ||
|
|
b328ad98fa | ||
|
|
d7d3bf5b2b | ||
|
|
896726a51b | ||
|
|
ea8a5b40e7 | ||
|
|
f38f218ae0 |
No files matched your search
@@ -10,6 +10,8 @@
|
||||
!backup.js
|
||||
!reading-analytics.js
|
||||
!reading-access.js
|
||||
!moments.js
|
||||
!reader-sessions.js
|
||||
!public/
|
||||
!public/**
|
||||
!greatest/
|
||||
@@ -22,6 +24,8 @@
|
||||
!scraper/**
|
||||
!peering/
|
||||
!peering/**
|
||||
!perf/
|
||||
!perf/**
|
||||
!tools/
|
||||
!tools/**
|
||||
scraper/.cache/**
|
||||
|
||||
@@ -20,6 +20,10 @@ READING_TIMEZONE=UTC
|
||||
READING_QUIZ_UNLOCK_MINUTES=60
|
||||
# Optional deployment-managed JSON file under DATA_PATH, e.g. /data/reading-quiz.json.
|
||||
# READING_QUIZ_CATALOG_PATH=/data/reading-quiz.json
|
||||
# Read an opened chapter into the OS cache ahead of the browser (1 = on, the default); see docs/performance.md.
|
||||
# READ_AHEAD=1
|
||||
# Performance trace from the start (also switchable at runtime: touch DATA_PATH/perf/trace.on).
|
||||
# PERF_TRACE=0
|
||||
PROXY_NETWORK=mangacircus-edge
|
||||
# Extra Compose files, ':'-separated (docs/deployment.md), e.g. HTTPS with the bundled Caddy:
|
||||
# COMPOSE_FILE=docker-compose.yml:deploy/compose.https.yml
|
||||
|
||||
@@ -12,6 +12,7 @@ Guide for contributors and coding agents. Users who only want to run it: [docs/d
|
||||
| `scraper/` | Download queue daemon (`service.js`, `daemon.js`), source adapters and parsers, the publisher that writes the library, metadata lookups |
|
||||
| `peering/` | Optional server-to-server sharing (gateway, client, adapter) |
|
||||
| `runtime/` | In-container supervisor and configuration |
|
||||
| `perf/` | Switchable performance trace and chapter read-ahead ([docs/performance.md](docs/performance.md)) |
|
||||
| `discovery/`, `greatest/` | Discovery Scout and the Greatest of All Time dashboard |
|
||||
| `tools/` | Demo library, health checks, maintenance tools (dry run by default), UI and Compose test drivers |
|
||||
| `deploy/` | `circus.sh` operations helper, HTTPS with Caddy |
|
||||
|
||||
+2
-1
@@ -3,7 +3,7 @@ RUN apk add --no-cache tzdata openssl
|
||||
WORKDIR /app
|
||||
ARG VCS_REF=development
|
||||
LABEL org.opencontainers.image.title="MangaCircus" org.opencontainers.image.revision=$VCS_REF
|
||||
COPY package.json server.js library.js reading-direction.js chapter-identity.js chapter-dates.js reading-analytics.js reading-access.js catalog.js backup.js ./
|
||||
COPY package.json server.js library.js reading-direction.js chapter-identity.js chapter-dates.js reading-analytics.js reading-access.js catalog.js backup.js moments.js reader-sessions.js ./
|
||||
COPY public ./public
|
||||
COPY greatest ./greatest
|
||||
COPY discovery ./discovery
|
||||
@@ -11,6 +11,7 @@ COPY tools ./tools
|
||||
COPY scraper ./scraper
|
||||
COPY peering ./peering
|
||||
COPY runtime ./runtime
|
||||
COPY perf ./perf
|
||||
RUN mkdir -p /library /data && chown node:node /data /library
|
||||
ENV PORT=8080 LIBRARY=/library DATA=/data TZ=UTC
|
||||
EXPOSE 8080
|
||||
|
||||
@@ -50,6 +50,10 @@ Settings has **User** and **Admin** menu groups. User sections cover Reading, Re
|
||||
Notifications and Account. Admin sections cover Scraper, Server & library, Users & access, Collections &
|
||||
metadata and Service health. Only admins see administration ([settings navigation](docs/settings-navigation.md)).
|
||||
|
||||
Reader settings (mode, zoom, page tone, skipping) are kept per sign-in: each device or browser keeps its own once you
|
||||
change one there, and a new sign-in starts from the one you used last. **Settings → Reading → Reader settings per
|
||||
session** shows every session's state and turns this off for one shared set ([details](docs/architecture.md#67-reader-settings-per-session)).
|
||||
|
||||
### Settings (environment variables)
|
||||
|
||||
These configure the reader process. In Docker most are set for you by `docker-compose.yml`; the variables
|
||||
@@ -63,6 +67,8 @@ you put in `.env` are listed in [docs/deployment.md](docs/deployment.md#4-config
|
||||
| `SIGNUP` | `closed` | `open` lets anyone create an account. Otherwise the admin adds users under Settings |
|
||||
| `SCAN_INTERVAL` | `10` | Minutes between shallow library change checks (`0` disables this timer) |
|
||||
| `FULL_SCAN_INTERVAL` | `1440` | Minutes between full library reconciliations (`0` disables this timer) |
|
||||
| `READ_AHEAD` | `1` | Read an opened chapter's pages (then the next chapter's) into the OS cache ahead of the browser; `0` = off ([docs/performance.md](docs/performance.md)) |
|
||||
| `PERF_TRACE` | `0` | `1` = performance trace from the start; it can also be switched at runtime with the file `DATA/perf/trace.on` ([docs/performance.md](docs/performance.md)) |
|
||||
| `SCAN_CONCURRENCY` | `8` | Chapter folders and images probed at once by an uncached scan (1–64); higher helps on network storage |
|
||||
| `COOKIE_SECURE` | – | `1` marks the login cookie `Secure` (use it when you serve over HTTPS) |
|
||||
| `SCRAPER_URL` | `http://127.0.0.1:8091` | Private scraper daemon URL |
|
||||
|
||||
@@ -61,6 +61,8 @@ services:
|
||||
DISCOVERY_HARVEST_AT: ${DISCOVERY_HARVEST_AT:-03:30}
|
||||
DISCOVERY_ANILIST_URL: ${DISCOVERY_ANILIST_URL:-}
|
||||
GREATEST_COVER_WARM: ${GREATEST_COVER_WARM:-1}
|
||||
READ_AHEAD: ${READ_AHEAD:-1}
|
||||
PERF_TRACE: ${PERF_TRACE:-0}
|
||||
networks: [backend, egress]
|
||||
healthcheck:
|
||||
test: [CMD, node, tools/healthcheck.js, reader]
|
||||
|
||||
+28
-5
@@ -130,11 +130,14 @@ This keeps ids human-readable and derivable without a database. The price: renam
|
||||
"readingStory": true, // show private retrospective summaries when history exists
|
||||
"continueRow": true, // library: "Continue where you left off" row at the top
|
||||
"zoom": { "flow": 1, "page": 1.5, "book": 1, "strip": 0.8 }, // page zoom per mode
|
||||
"webhook": ""
|
||||
}
|
||||
"webhook": "",
|
||||
"readerPerSession": true // reader settings per login session (section 6.7)
|
||||
},
|
||||
"readerFrom": "3f9c…" // public id of the session the latest reader settings came from
|
||||
}
|
||||
},
|
||||
"sessions": { "<sha256 of token>": { "u": "admin", "t": 1790796794000 } },
|
||||
"sessions": { "<sha256 of token>": { "u": "admin", "t": 1790796794000, "seen": 1790796794000,
|
||||
"agent": "<User-Agent>", "reader": { …locked reader settings, only after a change… } } },
|
||||
"progress": { "admin": { "Moon Garden": { "1/2": {
|
||||
"v": "c0002 Roots [MangaDex]", // version read
|
||||
"p": 4, "n": 14, // page index in that version (all pages), page count
|
||||
@@ -177,7 +180,7 @@ This keeps ids human-readable and derivable without a database. The price: renam
|
||||
|
||||
| Path | Handling |
|
||||
|------|----------|
|
||||
| `/api/*` | `routes["<METHOD> <path>"]`. Handlers get `{ u, b, q, req, res }` and return an object, which is sent as JSON. Throwing `HttpError(code, message, extra)` sends `{ error, ...extra }` with that status (for example `nextAllowedAt` on a discovery `429`) |
|
||||
| `/api/*` | `routes["<METHOD> <path>"]`. Handlers get `{ u, b, q, req, res, sess, sk }` (`sess` = the login session record, `sk` its key) and return an object, which is sent as JSON. Throwing `HttpError(code, message, extra)` sends `{ error, ...extra }` with that status (for example `nextAllowedAt` on a discovery `429`) |
|
||||
| `/lib/*` | Image files from `LIBRARY`. Requires a session, allows image extensions only, and the path must resolve inside `LIBRARY`. `ETag` + `private, max-age=86400` |
|
||||
| everything else | Static files from `public/` (`/` → `index.html`). `ETag` + `no-cache` |
|
||||
|
||||
@@ -194,7 +197,9 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js
|
||||
| `POST /api/login` `{name, password}` | public | Starts a session (cookie `sid`) |
|
||||
| `POST /api/logout` | user | Ends the session |
|
||||
| `GET /api/me` | user | Name, admin flag, settings, unread count |
|
||||
| `POST /api/settings` `{…partial}` | user | Validated update of the settings keys above |
|
||||
| `POST /api/settings` `{…partial}` | user | Validated update of the settings keys above; reader keys go to the current session (6.7) |
|
||||
| `GET /api/reader-sessions` | user | `{ enabled, current, latestFrom, latest, sessions: [{ id, device, created, seen, current, locked, reader }] }` |
|
||||
| `POST /api/reader-sessions` `{id, action}` | user | `follow`: the session drops its own reader settings and follows the latest; `latest`: its settings become the latest |
|
||||
| `POST /api/password` `{old, new}` | user | Change password |
|
||||
| `GET /api/library` | user | Manga list with covers, counts, the user's read count, unread notifications, last read, and optional personal Reading Story aggregates |
|
||||
| `GET /api/manga?m=` | user | Manga details, books, chapters with ranked versions, progress, optional chapter-time summaries, and `continue` target |
|
||||
@@ -210,6 +215,7 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js
|
||||
| `POST /api/moments` `{m, c, v, pages, layout, region, cat, note}` | user | Save a moment: `pages` are 1–4 distinct indexes in displayed order, `layout` `row` (default) or `column`, `cat` one of `twist hot awesome cute funny feels` (required), `region` fractions of the pages put together, or null |
|
||||
| `POST /api/moments/update` `{id, cat?, region?, note?}` · `POST /api/moments/delete` `{id}` | user | Edit or delete one of the caller's own moments (another user's id is `404`) |
|
||||
| `GET /api/notifications` · `POST /api/notifications/read` `{id \| all}` | user | Notification list and read state |
|
||||
| `POST /api/perf` `{page, events: […]}` | user | Browser timings for the performance trace; ignored with `{on: false}` while the trace is off. `/api/chapter` returns `perf: true` while it is on ([performance.md](performance.md)) |
|
||||
| `GET /api/admin` | admin | Users, last scan result, library path |
|
||||
| `POST /api/admin/rescan` `{full}` | admin | Scan now, and wait for the result |
|
||||
| `POST /api/admin/user` `{name, password, admin}` | admin | Create a user or reset a password |
|
||||
@@ -268,6 +274,22 @@ The reader alone talks to AniList for discovery ([discovery.md](discovery.md)).
|
||||
(all reasons, strongest first) next to `reason`; seed evidence carries `fetchedAt`.
|
||||
- `DISCOVERY_ANILIST_URL` overrides the AniList endpoint (tests point it at a local fixture server).
|
||||
|
||||
### 6.7 Reader settings per session
|
||||
|
||||
The reader keys (`startMode`, `lastMode`, `adapt`, `skipExtras`, `skipCovers`, `flowPageGap`, `pageTone`, `zoom`)
|
||||
can differ per login session, so a phone and a desktop keep their own mode and zoom (`reader-sessions.js`):
|
||||
|
||||
- `user.settings` holds the **latest** reader set. A session without its own set follows it.
|
||||
- The first reader change in a session copies the latest set into `session.reader` with the change applied: the session
|
||||
is now **locked** and keeps that set whatever other sessions do. The change also becomes the latest set.
|
||||
- Loading a page (`GET /api/me`) from a locked session makes its set the latest again, so a new sign-in starts from the
|
||||
session used last. A new sign-in has no set of its own until it changes something.
|
||||
- Other settings (Reading Story, dates, discovery, webhook, …) stay per user.
|
||||
- `readerPerSession: false` (Settings → Reading) brings back one shared set: changes go to `user.settings` and apply to
|
||||
every session. Locked sets are kept and apply again when the switch is turned back on.
|
||||
|
||||
Settings → Reading → *Reader settings per session* lists the user's sessions (device from the User-Agent, sign-in and last
|
||||
activity, own or following) with "Follow latest again" and "Use for new sessions".
|
||||
## 7. Frontend
|
||||
|
||||
### 7.1 Pages
|
||||
@@ -400,6 +422,7 @@ docker compose up -d --build
|
||||
| User data | `db.json` is rewritten as a whole, at most about twice per second while people read. Comfortable up to a few MB (a household or small group, thousands of chapters read) |
|
||||
| Concurrency | One process, one writer. No clustering or horizontal scaling by design |
|
||||
| Images | Served as stored. No resizing or thumbnails, so large covers make the library page heavy |
|
||||
| Slow storage | Opening a chapter reads its pages (then the next chapter's) into the OS cache, two at a time (`READ_AHEAD`). Where the time goes: [performance.md](performance.md) |
|
||||
|
||||
## 10. Design decisions
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# Performance trace, load test and read-ahead
|
||||
|
||||
When pages load slowly, the trace shows where the time goes: the storage (a NAS, a disk waking up), the
|
||||
reader process (CPU, event loop), library scans running at the same time, or the network to the browser.
|
||||
|
||||
## Switching the trace on and off
|
||||
|
||||
No restart is needed. The reader checks for a flag file every 5 seconds:
|
||||
|
||||
```bash
|
||||
touch "$DATA/perf/trace.on" # on (Docker: the data folder mounted at /data)
|
||||
rm "$DATA/perf/trace.on" # off
|
||||
```
|
||||
|
||||
`PERF_TRACE=1` in the environment keeps it on from the start. While it is off, the reader does no extra work.
|
||||
|
||||
The trace goes to `DATA/perf/trace-YYYY-MM-DD.jsonl` (UTC days), one JSON object per line. Files older than
|
||||
14 days are deleted; one day stops growing at 256 MB. The file contains request paths (manga and chapter
|
||||
names) and user names, so treat it like the rest of `DATA`.
|
||||
|
||||
## What is recorded
|
||||
|
||||
Every line has `t` (server time, ms since the epoch) and `k` (kind):
|
||||
|
||||
| `k` | Fields | Meaning |
|
||||
|---|---|---|
|
||||
| `req` | `m`, `path`, `status`, `ms`, `bytes`, `conn`, `aborted`; for files `stat`, `open`, `first`, `readEnd`, `disk`, `netWait`; for pages also `access` | One HTTP request. Phase fields are ms since the request arrived: `stat` file lookup done, `open` file opened, `first` first chunk read, `readEnd` last chunk read. `disk` = time spent waiting for the storage, `netWait` = time the file stream was paused because the client did not take data fast enough. `conn` > 1 means a reused keep-alive connection |
|
||||
| `client` | `e`: `hello`, `res`, `view`, `stall` | Reported by the reader page while the trace is on (`public/perf.js`). `res`: the browser's resource timing of a page image (`queue`, `dns`, `connect`, `ttfb`, `download`, `total`, `size`, `server` = the reader's `stat` time, `cached`). `view`: a page turn in page/book mode, `wait` until the new page was decoded and painted, `preloaded`. `stall`: flow mode, a page on screen before its image, `wait` until it loaded. `hello`: browser, viewport, connection type |
|
||||
| `nas` | `statMs`, `firstMs`, `ms`, `bytes` or `error` | Every `PERF_NAS_PROBE_SECONDS` (60) one random library page is read. Out of a large library it is almost never cached, so this is the storage's cold latency over time; seconds after a quiet period mean disks were in standby |
|
||||
| `loop` | `lagP50`, `lagP99`, `lagMax`, `cpu`, `rss`, `heap` | Every 5 s: event-loop delay (ms), CPU % of one core, memory (MB) |
|
||||
| `scan` | `mode`, `ms`, `changed` | Library scans (change checks every `SCAN_INTERVAL`, full reconciliations) |
|
||||
| `ahead` | `tag`, `files`, `done`, `skipped`, `bytes`, `ms`, `replaced` | A read-ahead batch (below) |
|
||||
| `scraper` | `endpoint`, `ms`, `status`/`error` | Calls from the reader to the scraper daemon |
|
||||
| `trace` | `on` | The trace was switched on or off |
|
||||
|
||||
## Reading the result
|
||||
|
||||
```bash
|
||||
node tools/perf-report.js "$DATA/perf" # all days
|
||||
node tools/perf-report.js "$DATA/perf" --since 2026-10-29T18:00Z --slow 300
|
||||
docker compose exec reader node tools/perf-report.js /data/perf # inside the container
|
||||
```
|
||||
|
||||
The report starts with a verdict (which phase dominates the slow loads), then percentiles for the server's
|
||||
phases, the browser's timings, page turns, flow stalls, the storage probe, the event loop, scans, read-ahead
|
||||
and the API routes. `--json` prints the same numbers for scripts and agents.
|
||||
|
||||
How to read it:
|
||||
|
||||
- **Slow `firstRead` / `readRest`, slow `nas` probes**: the storage is slow. Probes of several seconds after
|
||||
minutes without reads point at disk standby (on a Synology: *Control Panel → Hardware & Power → HDD
|
||||
Hibernation*).
|
||||
- **Slow `stat`**: metadata lookups on the storage are slow (often during a scan, see `scan`).
|
||||
- **High `netWait`, or browser `download` much longer than the server's `ms`**: the network between server
|
||||
and browser (Wi-Fi, VPN, a proxy).
|
||||
- **Browser `queue`**: the browser waited for a free connection (too many parallel requests).
|
||||
- **High `lagMax`**: the reader process was busy and could not answer.
|
||||
|
||||
## Load test
|
||||
|
||||
`tools/perf-bench.js` simulates readers paging through random chapters with a think time and a small
|
||||
look-ahead, and reports how long a reader looked at an empty page (`wait`), time to first byte and total
|
||||
per page. It only reads (no progress is written). Use a separate account:
|
||||
|
||||
```bash
|
||||
BENCH_PASSWORD=… node tools/perf-bench.js --url http://server:8080 --user bench --password-env BENCH_PASSWORD \
|
||||
--readers 4 --chapters 3 --pages 40 --think 400 --ahead 2
|
||||
```
|
||||
|
||||
Run it with the trace on to see the server's side of the same requests.
|
||||
|
||||
## Read-ahead
|
||||
|
||||
`READ_AHEAD=1` (default): when a chapter is opened, the reader reads its pages once in page order, starting
|
||||
at the saved page, two at a time, and then the next chapter's. The data is thrown away; the OS page cache
|
||||
keeps it, so the browser's requests are answered from memory instead of the storage. Opening another
|
||||
chapter replaces whatever is still queued. `READ_AHEAD=0` turns it off (useful on a local SSD, or to compare
|
||||
with the trace).
|
||||
@@ -199,7 +199,8 @@ All responses are JSON. When `SCRAPER_TOKEN` is set, send
|
||||
| Method | Path | Body / result |
|
||||
|---|---|---|
|
||||
| `GET` | `/health` | Basic liveness |
|
||||
| `GET` | `/api/state` | Queue, all active source checkpoints, settings, errors, history, adapters |
|
||||
| `GET` | `/api/state` | Queue, all active source checkpoints, settings, errors, history, adapters. The library summary in it is rebuilt in the background after a publish, or when it is 15 minutes old (the walk reads every `chapter.json`) |
|
||||
| `GET` | `/api/revision` | `{ publishedRevision, status, paused }` only; the reader polls this every 15 s |
|
||||
| `POST` | `/api/queue` | `{url,title?,metadataSearch?,scope?,existingPolicy?,libraryMangaId?,inspect?}` |
|
||||
| `POST` | `/api/queue/remove` | `{id}`; active downloads cannot be removed |
|
||||
| `POST` | `/api/queue/clear-completed` | `{source?}`; remove inactive successful download job rows only; preserve library, history, checkpoints and other statuses |
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
'use strict';
|
||||
/*
|
||||
* Read-ahead for network libraries: when a chapter is opened, its pages (then the next chapter's) are read
|
||||
* once in page order, a few at a time, so the OS page cache holds them before the browser asks. On a NAS
|
||||
* with spinning disks a cold page costs 10-100 ms (seconds while a disk wakes up); a cached one well under
|
||||
* 5 ms. The data is discarded; only the cache effect matters.
|
||||
*
|
||||
* One queue per reader (`key`, the user): opening another chapter replaces that reader's queue only, and
|
||||
* the queues take turns so two people reading at once both get their pages ahead.
|
||||
*/
|
||||
const fs = require('fs');
|
||||
|
||||
class ReadAhead {
|
||||
constructor({ concurrency = 2, remember = 4000, onBatch = () => {} } = {}) {
|
||||
this.concurrency = concurrency; this.remember = remember; this.onBatch = onBatch;
|
||||
this.batches = new Map(); this.turn = []; this.active = 0; this.recent = new Map(); this.buffers = [];
|
||||
}
|
||||
|
||||
// `files` in reading order; `tag` names the batch in the trace.
|
||||
warm(files, tag, key = '') {
|
||||
const old = this.batches.get(key);
|
||||
if (old) this.finish(key, old, true);
|
||||
const queue = files.filter(f => !this.recent.has(f));
|
||||
const batch = { tag, queue, files: queue.length, skipped: files.length - queue.length, bytes: 0, done: 0, active: 0, t0: performance.now() };
|
||||
this.batches.set(key, batch);
|
||||
if (!this.turn.includes(key)) this.turn.unshift(key); // a fresh request goes next
|
||||
if (!queue.length) this.finish(key, batch, false);
|
||||
this.pump();
|
||||
}
|
||||
// Runs after the reader's current batch (e.g. the next chapter) unless a newer warm() replaces it first.
|
||||
then(files, tag, key = '') {
|
||||
const batch = this.batches.get(key);
|
||||
if (!batch) this.warm(files, tag, key); else batch.next = () => this.warm(files, tag, key);
|
||||
}
|
||||
|
||||
pump() {
|
||||
while (this.active < this.concurrency) {
|
||||
const key = this.turn.find(k => this.batches.get(k)?.queue.length);
|
||||
if (key === undefined) return;
|
||||
this.turn.push(...this.turn.splice(this.turn.indexOf(key), 1)); // round robin between readers
|
||||
const batch = this.batches.get(key), file = batch.queue.shift();
|
||||
this.active++; batch.active++;
|
||||
this.readFile(file).then(bytes => { this.note(file); batch.bytes += bytes; batch.done++; }, () => {}).finally(() => {
|
||||
this.active--; batch.active--;
|
||||
if (this.batches.get(key) === batch && !batch.queue.length && !batch.active) this.finish(key, batch, false);
|
||||
this.pump();
|
||||
});
|
||||
}
|
||||
}
|
||||
finish(key, b, replaced) {
|
||||
if (this.batches.get(key) === b) this.batches.delete(key);
|
||||
if (!this.batches.has(key)) this.turn = this.turn.filter(k => k !== key);
|
||||
b.queue = [];
|
||||
this.onBatch({ tag: b.tag, files: b.files, done: b.done, skipped: b.skipped, bytes: b.bytes,
|
||||
ms: Math.round(performance.now() - b.t0), replaced });
|
||||
if (!replaced && b.next) b.next();
|
||||
}
|
||||
note(file) {
|
||||
this.recent.delete(file); this.recent.set(file, 1);
|
||||
while (this.recent.size > this.remember) this.recent.delete(this.recent.keys().next().value);
|
||||
}
|
||||
|
||||
async readFile(file) {
|
||||
const buf = this.buffers.pop() || Buffer.allocUnsafe(256 * 1024);
|
||||
const fh = await fs.promises.open(file, 'r');
|
||||
let total = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const { bytesRead } = await fh.read(buf, 0, buf.length, total);
|
||||
if (!bytesRead) break;
|
||||
total += bytesRead;
|
||||
}
|
||||
} finally { await fh.close(); this.buffers.push(buf); }
|
||||
return total;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { ReadAhead };
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
'use strict';
|
||||
/*
|
||||
* Switchable performance trace: one JSON object per line in DATA/perf/trace-YYYY-MM-DD.jsonl.
|
||||
*
|
||||
* On while PERF_TRACE=1 or while the file DATA/perf/trace.on exists (checked every few seconds, so it can be
|
||||
* switched on a running server with `touch` / `rm`). Off, every call is a cheap no-op.
|
||||
*
|
||||
* Event kinds (field `k`): `req` one HTTP request with its phases, `client` timings the reader page reports,
|
||||
* `loop` event-loop lag / CPU / memory every SAMPLE_MS, `nas` a periodic read of one uncached library page,
|
||||
* `scan` library scans, `scraper` calls to the scraper daemon, `ahead` read-ahead batches, `trace` on/off.
|
||||
* The format and the report tool: docs/performance.md, tools/perf-report.js.
|
||||
*/
|
||||
const fs = require('fs'), path = require('path'), { monitorEventLoopDelay } = require('perf_hooks');
|
||||
|
||||
const CHECK_MS = 5000, FLUSH_MS = 1000, SAMPLE_MS = 5000;
|
||||
const MAX_DAY_BYTES = 256 * 1024 * 1024; // a runaway trace never fills the data disk
|
||||
const KEEP_DAYS = 14;
|
||||
|
||||
class Tracer {
|
||||
constructor({ dir, env = process.env, now = Date.now } = {}) {
|
||||
this.dir = dir; this.now = now;
|
||||
this.forced = env.PERF_TRACE === '1';
|
||||
this.flag = path.join(dir, 'trace.on');
|
||||
this.on = false; this.buf = []; this.day = ''; this.dayBytes = 0; this.dropped = 0;
|
||||
this.timers = []; this.probe = null; this.probeMs = Math.max(10, +env.PERF_NAS_PROBE_SECONDS || 60) * 1000;
|
||||
}
|
||||
|
||||
start() {
|
||||
this.check();
|
||||
this.timers.push(setInterval(() => this.check(), CHECK_MS).unref());
|
||||
this.timers.push(setInterval(() => this.flush(), FLUSH_MS).unref());
|
||||
return this;
|
||||
}
|
||||
stop() { for (const t of this.timers) clearInterval(t); this.timers = []; this.setOn(false); this.flush(); }
|
||||
|
||||
check() {
|
||||
let on = this.forced;
|
||||
if (!on) try { fs.accessSync(this.flag); on = true; } catch { /* off */ }
|
||||
this.setOn(on);
|
||||
}
|
||||
setOn(on) {
|
||||
if (on === this.on) return;
|
||||
if (on) {
|
||||
this.on = true;
|
||||
this.log({ k: 'trace', on: true, pid: process.pid, node: process.version });
|
||||
this.startSampler();
|
||||
} else {
|
||||
this.log({ k: 'trace', on: false });
|
||||
this.on = false;
|
||||
this.stopSampler();
|
||||
}
|
||||
}
|
||||
|
||||
log(event) {
|
||||
if (!this.on) return;
|
||||
this.buf.push(JSON.stringify({ t: this.now(), ...event }));
|
||||
if (this.buf.length >= 500) this.flush();
|
||||
}
|
||||
|
||||
flush() {
|
||||
if (!this.buf.length) return;
|
||||
const lines = this.buf.join('\n') + '\n'; this.buf = [];
|
||||
const day = new Date(this.now()).toISOString().slice(0, 10);
|
||||
try {
|
||||
if (day !== this.day) { this.day = day; this.dayBytes = 0; fs.mkdirSync(this.dir, { recursive: true }); this.prune(); }
|
||||
if (this.dayBytes + lines.length > MAX_DAY_BYTES) { this.dropped += lines.length; return; }
|
||||
fs.appendFileSync(path.join(this.dir, `trace-${day}.jsonl`), lines);
|
||||
this.dayBytes += lines.length;
|
||||
} catch (error) { console.warn('[perf] trace write failed:', error.message); }
|
||||
}
|
||||
prune() {
|
||||
const cutoff = new Date(this.now() - KEEP_DAYS * 864e5).toISOString().slice(0, 10);
|
||||
for (const name of fs.readdirSync(this.dir)) {
|
||||
const m = /^trace-(\d{4}-\d\d-\d\d)\.jsonl$/.exec(name);
|
||||
if (m && m[1] < cutoff) fs.rmSync(path.join(this.dir, name), { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// Event-loop delay shows whether the reader itself (CPU, sync work) holds requests back.
|
||||
startSampler() {
|
||||
const h = monitorEventLoopDelay({ resolution: 10 }); h.enable();
|
||||
let cpu = process.cpuUsage(), at = process.hrtime.bigint();
|
||||
this.sampler = { h, timer: setInterval(() => {
|
||||
const nowCpu = process.cpuUsage(), nowAt = process.hrtime.bigint(), wall = Number(nowAt - at) / 1000;
|
||||
const mem = process.memoryUsage();
|
||||
this.log({ k: 'loop', lagP50: ms(h.percentile(50)), lagP99: ms(h.percentile(99)), lagMax: ms(h.max),
|
||||
cpu: round(((nowCpu.user - cpu.user) + (nowCpu.system - cpu.system)) / wall * 100), rss: mb(mem.rss), heap: mb(mem.heapUsed) });
|
||||
h.reset(); cpu = nowCpu; at = nowAt;
|
||||
}, SAMPLE_MS).unref() };
|
||||
this.startProbe();
|
||||
}
|
||||
stopSampler() {
|
||||
if (this.sampler) { clearInterval(this.sampler.timer); this.sampler.h.disable(); this.sampler = null; }
|
||||
if (this.probe) { clearInterval(this.probe); this.probe = null; }
|
||||
}
|
||||
|
||||
// NAS probe: read one random library page now and then. A page picked at random out of a large library is
|
||||
// almost never in the OS cache, so this measures the storage itself (and shows disks waking from standby).
|
||||
setProbeSource(pick) { this.pick = pick; if (this.on) this.startProbe(); }
|
||||
startProbe() {
|
||||
if (this.probe || !this.pick) return;
|
||||
const run = () => this.probeOnce().catch(() => {});
|
||||
this.probe = setInterval(run, this.probeMs).unref();
|
||||
setTimeout(run, 1000).unref();
|
||||
}
|
||||
async probeOnce() {
|
||||
const file = this.pick && this.pick();
|
||||
if (!file) return;
|
||||
const t0 = performance.now();
|
||||
let st, bytes = 0, firstMs = null, error = null;
|
||||
try {
|
||||
st = await fs.promises.stat(file);
|
||||
const statMs = performance.now() - t0;
|
||||
const fh = await fs.promises.open(file, 'r');
|
||||
try {
|
||||
const chunk = Buffer.allocUnsafe(256 * 1024);
|
||||
for (;;) {
|
||||
const { bytesRead } = await fh.read(chunk, 0, chunk.length, bytes);
|
||||
if (firstMs === null) firstMs = performance.now() - t0;
|
||||
if (!bytesRead) break;
|
||||
bytes += bytesRead;
|
||||
}
|
||||
} finally { await fh.close(); }
|
||||
this.log({ k: 'nas', statMs: round(statMs), firstMs: round(firstMs), ms: round(performance.now() - t0), bytes });
|
||||
} catch (e) { error = e.code || e.message; this.log({ k: 'nas', ms: round(performance.now() - t0), error }); }
|
||||
}
|
||||
}
|
||||
|
||||
// Wraps one HTTP request: `mark(name)` stores the time since the request arrived, `set` adds fields, and the
|
||||
// event is written when the response is done (or the client went away). Off, returns a shared no-op.
|
||||
const NOOP = { mark() {}, set() {}, stream() {}, elapsed: () => 0, on: false };
|
||||
function requestTrace(tracer, req, res) {
|
||||
if (!tracer.on) return NOOP;
|
||||
const t0 = performance.now(), ev = { k: 'req', m: req.method, path: req.url.split('?')[0].slice(0, 300) };
|
||||
const trace = {
|
||||
on: true,
|
||||
mark(name) { ev[name] = round(performance.now() - t0); },
|
||||
elapsed: () => round(performance.now() - t0),
|
||||
set(fields) { Object.assign(ev, fields); },
|
||||
// Splits a file stream's time into waiting for the disk and waiting for the client (TCP backpressure):
|
||||
// pipe() pauses the file stream while the socket buffer is full and resumes it on 'drain'.
|
||||
stream(src) {
|
||||
let pausedAt = 0, waitNet = 0;
|
||||
src.on('open', () => trace.mark('open'));
|
||||
src.once('data', () => trace.mark('first'));
|
||||
src.on('data', chunk => { ev.bytes = (ev.bytes || 0) + chunk.length; });
|
||||
src.on('pause', () => { pausedAt = performance.now(); });
|
||||
src.on('resume', () => { if (pausedAt) { waitNet += performance.now() - pausedAt; pausedAt = 0; } });
|
||||
src.on('end', () => {
|
||||
trace.mark('readEnd');
|
||||
ev.netWait = round(waitNet);
|
||||
ev.disk = round(Math.max(0, ev.readEnd - (ev.open || 0) - waitNet));
|
||||
});
|
||||
},
|
||||
};
|
||||
let done = false;
|
||||
const finish = aborted => {
|
||||
if (done) return; done = true;
|
||||
ev.ms = round(performance.now() - t0); ev.status = res.statusCode;
|
||||
if (aborted) ev.aborted = true;
|
||||
ev.conn = req.socket._perfRequests = (req.socket._perfRequests || 0) + 1; // >1: keep-alive reuse
|
||||
tracer.log(ev);
|
||||
};
|
||||
res.on('finish', () => finish(false));
|
||||
res.on('close', () => finish(!res.writableFinished));
|
||||
return trace;
|
||||
}
|
||||
|
||||
const round = v => (v == null || Number.isNaN(v) ? null : Math.round(v * 10) / 10);
|
||||
const ms = ns => round(ns / 1e6);
|
||||
const mb = b => Math.round(b / 1048576);
|
||||
|
||||
module.exports = { Tracer, requestTrace };
|
||||
@@ -387,6 +387,8 @@ main { max-width: 1200px; margin: 0 auto; padding: 16px; }
|
||||
.login input { width: 100%; }
|
||||
.saved { color: #7fd3a0; font-size: 13px; }
|
||||
.reading-data { margin-top: 16px; padding-top: 12px; border-top: 1px solid var(--line); }
|
||||
.reader-sessions { list-style: none; margin: 12px 0 0; padding: 0; }
|
||||
.reader-sessions li { display: grid; gap: 4px; padding: 10px 0; border-top: 1px solid var(--line); overflow-wrap: anywhere; }
|
||||
.reading-data > summary { cursor: pointer; font-weight: 600; }
|
||||
.reading-danger { margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--line); }
|
||||
.reading-danger p { margin: 3px 0 8px; }
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
'use strict';
|
||||
/*
|
||||
* Reader-side half of the performance trace (docs/performance.md). Only started when the server says its
|
||||
* trace is on (`perf: true` in /api/chapter). It reports, in batches to POST api/perf:
|
||||
* - `res` the browser's resource timing of every library image: queueing, connection, time to first byte,
|
||||
* download, size, the server's own `stat` time and whether the connection was reused;
|
||||
* - `view` a page turn (page/book mode): how long the new page(s) took to appear, and whether they were
|
||||
* preloaded already;
|
||||
* - `stall` flow mode: a page scrolled into view before its image had loaded, and how long it stayed empty.
|
||||
*/
|
||||
window.PerfTrace = {
|
||||
start(pageName) {
|
||||
let queue = [], on = true;
|
||||
const push = e => { if (on) queue.push(e); };
|
||||
const rel = url => { try { return decodeURIComponent(new URL(url).pathname); } catch { return String(url); } };
|
||||
const flush = beacon => {
|
||||
if (!queue.length || !on) return;
|
||||
const body = JSON.stringify({ page: pageName, events: queue.splice(0, 300) });
|
||||
if (beacon && navigator.sendBeacon) navigator.sendBeacon('api/perf', new Blob([body], { type: 'application/json' }));
|
||||
else fetch('api/perf', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body, keepalive: true })
|
||||
.then(r => r.json()).then(r => { if (r.on === false) on = false; }).catch(() => {});
|
||||
};
|
||||
setInterval(() => flush(false), 5000);
|
||||
addEventListener('pagehide', () => flush(true));
|
||||
push({ e: 'hello', ua: navigator.userAgent, conn: navigator.connection ? navigator.connection.effectiveType : '',
|
||||
dpr: devicePixelRatio, vw: innerWidth, vh: innerHeight });
|
||||
|
||||
if (window.PerformanceObserver) {
|
||||
new PerformanceObserver(list => {
|
||||
for (const r of list.getEntries()) {
|
||||
if (!/\/lib\//.test(r.name)) continue;
|
||||
const st = (r.serverTiming || []).find(x => x.name === 'stat');
|
||||
push({ e: 'res', url: rel(r.name), at: r.startTime,
|
||||
queue: r.requestStart ? r.requestStart - r.startTime : null, // waiting for a free connection
|
||||
connect: r.connectEnd - r.connectStart, dns: r.domainLookupEnd - r.domainLookupStart,
|
||||
ttfb: r.responseStart && r.requestStart ? r.responseStart - r.requestStart : null,
|
||||
download: r.responseStart ? r.responseEnd - r.responseStart : null,
|
||||
total: r.duration, size: r.transferSize, body: r.encodedBodySize, server: st ? st.duration : null,
|
||||
proto: r.nextHopProtocol || '', cached: r.transferSize === 0 && r.encodedBodySize > 0 });
|
||||
}
|
||||
}).observe({ type: 'resource', buffered: true });
|
||||
}
|
||||
|
||||
return {
|
||||
// Called right after a page turn put new <img> elements on screen.
|
||||
view(imgs, info) {
|
||||
const t0 = performance.now(), waiting = imgs.filter(img => !img.complete);
|
||||
const event = { e: 'view', ...info, n: imgs.length, preloaded: !waiting.length };
|
||||
if (!waiting.length) { push({ ...event, wait: 0 }); return; }
|
||||
Promise.all(waiting.map(img => (img.decode ? img.decode() : Promise.resolve()).catch(() => {})))
|
||||
.then(() => requestAnimationFrame(() => push({ ...event, wait: performance.now() - t0,
|
||||
url: rel(waiting[0].currentSrc || waiting[0].src) })));
|
||||
},
|
||||
// Flow mode: watch pages enter the real viewport (not the preload margin) before their image is there.
|
||||
watchFlow(root, els) {
|
||||
if (!window.IntersectionObserver) return;
|
||||
const io = new IntersectionObserver(entries => {
|
||||
for (const en of entries) {
|
||||
const img = en.target.firstChild;
|
||||
if (!en.isIntersecting || (img.complete && img.naturalWidth)) continue;
|
||||
const t0 = performance.now(), hadSrc = !!img.getAttribute('src');
|
||||
const done = () => push({ e: 'stall', wait: performance.now() - t0, hadSrc, url: rel(img.currentSrc || img.src) });
|
||||
img.addEventListener('load', done, { once: true });
|
||||
io.unobserve(en.target);
|
||||
}
|
||||
}, { root });
|
||||
for (const el of els) io.observe(el);
|
||||
return io;
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -16,6 +16,7 @@
|
||||
<script src="reading-clock.js" defer></script>
|
||||
<script src="keys.js" defer></script>
|
||||
<script src="moments.js" defer></script>
|
||||
<script src="perf.js" defer></script>
|
||||
<script src="reader.js" defer></script>
|
||||
</head>
|
||||
<body class="reader">
|
||||
|
||||
@@ -74,6 +74,9 @@
|
||||
D = payload;
|
||||
}
|
||||
catch (e) { canvas.textContent = ''; canvas.append(h('p', { class: 'loading' }, e.message + ' · ', h('a', { href: './' }, 'Library'))); return; }
|
||||
// Timings for the server's performance trace, only while an admin has it switched on.
|
||||
const perf = D.perf && window.PerfTrace ? PerfTrace.start('read') : null;
|
||||
let perfFlow = null;
|
||||
|
||||
// An already-open chapter also closes when a window ends or an unlock expires.
|
||||
const checkReadingAccess = async () => {
|
||||
@@ -250,11 +253,13 @@
|
||||
if (mode === 'flow') {
|
||||
els = list.map(p => canvas.appendChild(pageEl(p, true)));
|
||||
endEl = canvas.appendChild(endCard());
|
||||
if (perf) { if (perfFlow) perfFlow.disconnect(); perfFlow = perf.watchFlow(stage, els); }
|
||||
} else if (atEnd) {
|
||||
endEl = canvas.appendChild(endCard());
|
||||
} else {
|
||||
shown = mode === 'page' ? [cur] : spreads[spreadIdx];
|
||||
els = shown.map(k => canvas.appendChild(pageEl(list[k])));
|
||||
if (perf) perf.view(els.map(el => el.firstChild), { mode, page: list[shown[0]].i });
|
||||
// The pages on screen get the bandwidth first, then the next ones are fetched.
|
||||
const token = ++buildToken, waiting = els.map(el => el.firstChild).filter(img => !img.complete);
|
||||
Promise.all(waiting.map(img => new Promise(done => {
|
||||
|
||||
@@ -15,11 +15,61 @@
|
||||
const saved = h('span', { class: 'saved', hidden: true }, 'Saved');
|
||||
shell.titleRow.append(saved);
|
||||
let savedTimer = 0;
|
||||
let refreshSessions = () => {};
|
||||
async function save(patch) {
|
||||
Object.assign(S, (await api('api/settings', patch)).settings);
|
||||
saved.hidden = false;
|
||||
clearTimeout(savedTimer);
|
||||
savedTimer = setTimeout(() => { saved.hidden = true; }, 1500);
|
||||
refreshSessions();
|
||||
}
|
||||
|
||||
// Which login session holds which reader settings (reader-sessions.js on the server).
|
||||
function readerSessionsCard() {
|
||||
const TONE = { original: 'Original', inverted: 'Inverted', sepia: 'Sepia', dim: 'Dim', 'extra-dim': 'Extra dim' };
|
||||
const START = { last: 'last mode', flow: 'always Flow', page: 'always Page', book: 'always Book' };
|
||||
const summary = r => [`${LABEL[r.lastMode] || r.lastMode} (opens: ${START[r.startMode] || r.startMode})`, `tone ${TONE[r.pageTone] || r.pageTone}`,
|
||||
'zoom ' + (Object.keys(r.zoom || {}).length ? Object.entries(r.zoom).map(([k, z]) => `${LABEL[k] || k} ${Math.round(z * 100)}%`).join(', ') : 'fit'),
|
||||
r.skipExtras ? 'skips extras' : null, r.skipCovers ? 'skips covers' : null, r.flowPageGap === false ? 'no Flow gap' : null].filter(Boolean).join(' · ');
|
||||
const state = h('div', {}, h('p', { class: 'muted' }, 'Loading sessions…'));
|
||||
const message = h('span', { class: 'small' });
|
||||
async function act(id, action, current) {
|
||||
try {
|
||||
draw(await api('api/reader-sessions', { id, action }));
|
||||
// This page's controls show this session's settings; reload them when they changed.
|
||||
if (current && action === 'follow') location.reload();
|
||||
} catch (error) { message.textContent = error.message; message.className = 'small error'; }
|
||||
}
|
||||
function draw(o) {
|
||||
const current = o.sessions.find(s => s.current), source = o.sessions.find(s => s.id === o.latestFrom);
|
||||
const latestText = !o.latestFrom ? 'one shared set (changed while per-session settings were off, or never changed)'
|
||||
: source ? (source.current ? 'this session' : `${source.device}, active ${ago(source.seen)}`) : 'a session that has since signed out';
|
||||
const intent = !o.enabled ? 'Off: every session uses the same reader settings, and a change applies everywhere.'
|
||||
: current?.locked ? 'This session has its own reader settings. Changes here stay here and become the starting point for new sessions.'
|
||||
: 'This session follows the latest reader settings. Your first change here gives it its own copy, which stays put when other sessions change theirs.';
|
||||
state.replaceChildren(
|
||||
h('p', {}, h('b', {}, 'Now: '), intent),
|
||||
o.enabled ? h('p', { class: 'muted small' }, 'New sessions start from the latest settings: ', h('b', {}, summary(o.latest)), ` — from ${latestText}.`)
|
||||
: h('p', { class: 'muted small' }, 'Shared settings: ', h('b', {}, summary(o.latest))),
|
||||
h('ul', { class: 'reader-sessions' }, o.sessions.map(s => h('li', {},
|
||||
h('div', {}, h('b', {}, s.device), s.current ? ' (this session)' : '', ' · ',
|
||||
h('span', { class: 'muted small' }, `signed in ${new Date(s.created).toLocaleDateString()}, active ${ago(s.seen)}`)),
|
||||
h('div', { class: 'small' }, !o.enabled ? 'Uses the shared settings' + (s.locked ? ' (its own copy is kept for when this is on again)' : '')
|
||||
: s.locked ? 'Own settings: ' + summary(s.reader) : 'Follows the latest settings'),
|
||||
o.enabled ? h('div', { class: 'row' },
|
||||
s.locked ? h('button', { class: 'btn small', onclick: () => act(s.id, 'follow', s.current) }, 'Follow latest again') : null,
|
||||
s.locked && s.id !== o.latestFrom ? h('button', { class: 'btn small', onclick: () => act(s.id, 'latest', s.current) }, 'Use for new sessions') : null) : null))),
|
||||
message);
|
||||
}
|
||||
refreshSessions = () => api('api/reader-sessions').then(draw, error => { message.textContent = error.message; message.className = 'small error'; });
|
||||
refreshSessions();
|
||||
return h('section', { class: 'card' },
|
||||
h('h2', {}, 'Reader settings per session'),
|
||||
// The controls above switch between this session's set and the shared one, so reload them.
|
||||
h('label', { class: 'check' },
|
||||
h('input', { type: 'checkbox', checked: !!S.readerPerSession, onchange: async e => { await save({ readerPerSession: e.target.checked }); location.reload(); } }),
|
||||
h('span', {}, 'Keep reader settings per session', h('div', { class: 'muted small' }, 'Each sign-in (device or browser) keeps its own reader mode, zoom, tone and the other reader preferences above once you change one there. A new sign-in copies the settings of the session you used last. Turn this off for one shared set.'))),
|
||||
state);
|
||||
}
|
||||
const check = (key, text, hint) => h('label', { class: 'check' },
|
||||
h('input', { type: 'checkbox', checked: !!S[key], onchange: e => save({ [key]: e.target.checked }) }),
|
||||
@@ -90,6 +140,8 @@
|
||||
h('button', { class: 'btn small', onclick: async () => { await save({ zoom: {} }); zoomText.textContent = zooms(); } }, 'Reset to fit'),
|
||||
h('div', { class: 'muted small' }, '100% = the page fits the screen. Change it in the reader with Ctrl + mouse wheel, a pinch, or the + / - keys.'))));
|
||||
|
||||
section('user/reading').append(readerSessionsCard());
|
||||
|
||||
section('user/reading').append(h('section', { class: 'card' },
|
||||
h('h2', {}, 'Library page'),
|
||||
check('continueRow', 'Show "Continue where you left off" at the top',
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
'use strict';
|
||||
/* Reader settings per login session (docs/architecture.md, "Reader settings per session").
|
||||
*
|
||||
* The user's own reader settings (`user.settings`) are the "latest" set. A session that never changed a
|
||||
* reader setting follows the latest set; the first change copies the latest set into the session and
|
||||
* locks it there, and also makes it the new latest set. Using a locked session makes its set the latest
|
||||
* again, so a new login starts from the last active session. With `readerPerSession: false` there is
|
||||
* one set per user, as before; locked sets are kept and return when the switch is turned on again. */
|
||||
const crypto = require('crypto');
|
||||
|
||||
const READER_KEYS = ['startMode', 'lastMode', 'adapt', 'skipExtras', 'skipCovers', 'flowPageGap', 'pageTone', 'zoom'];
|
||||
|
||||
const perSession = settings => settings.readerPerSession !== false;
|
||||
const publicId = key => crypto.createHash('sha256').update('reader-session:' + key).digest('hex').slice(0, 16);
|
||||
|
||||
function pick(settings) {
|
||||
const out = {};
|
||||
for (const k of READER_KEYS) if (settings[k] !== undefined) out[k] = k === 'zoom' ? { ...settings[k] } : settings[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
// `base` is the user's merged settings (defaults + user.settings); the session's locked set wins over it.
|
||||
function effective(base, session) {
|
||||
if (!perSession(base) || !session?.reader) return base;
|
||||
return { ...base, ...pick(session.reader), zoom: { ...session.reader.zoom } };
|
||||
}
|
||||
|
||||
// Splits a validated settings patch: reader keys go to the session (locking it) and to the latest set.
|
||||
function applyPatch({ user, base, session, key, patch }) {
|
||||
const s = (user.settings = user.settings || {});
|
||||
const reader = pick(patch), rest = Object.fromEntries(Object.entries(patch).filter(([k]) => !READER_KEYS.includes(k)));
|
||||
Object.assign(s, rest);
|
||||
if (!Object.keys(reader).length) return;
|
||||
if (perSession({ ...base, ...rest }) && session) {
|
||||
session.reader = { ...pick(effective(base, session)), ...reader };
|
||||
Object.assign(s, pick(session.reader));
|
||||
user.readerFrom = publicId(key);
|
||||
} else {
|
||||
Object.assign(s, reader);
|
||||
user.readerFrom = null;
|
||||
}
|
||||
}
|
||||
|
||||
// A request from a locked session makes its set the latest one. Returns true when something changed.
|
||||
function noteActive(user, base, session, key) {
|
||||
if (!perSession(base) || !session?.reader) return false;
|
||||
const id = publicId(key);
|
||||
if (user.readerFrom === id) return false;
|
||||
user.settings = { ...(user.settings || {}), ...pick(session.reader) };
|
||||
user.readerFrom = id;
|
||||
return true;
|
||||
}
|
||||
|
||||
function deviceLabel(agent) {
|
||||
const a = String(agent || '');
|
||||
if (!a) return 'Unknown device';
|
||||
const os = /iPhone/.test(a) ? 'iPhone' : /iPad/.test(a) ? 'iPad' : /Android/.test(a) ? 'Android'
|
||||
: /Windows/.test(a) ? 'Windows' : /Mac OS X|Macintosh/.test(a) ? 'Mac' : /CrOS/.test(a) ? 'ChromeOS' : /Linux/.test(a) ? 'Linux' : '';
|
||||
const browser = /Edg\//.test(a) ? 'Edge' : /OPR\//.test(a) ? 'Opera' : /Firefox\//.test(a) ? 'Firefox'
|
||||
: /Chrome\//.test(a) ? 'Chrome' : /Safari\//.test(a) ? 'Safari' : '';
|
||||
return [browser, os].filter(Boolean).join(' on ') || a.slice(0, 60);
|
||||
}
|
||||
|
||||
// What Settings shows: every login session of the user with its state and effective reader set.
|
||||
function overview(user, base, sessions, currentKey) {
|
||||
const list = Object.entries(sessions).filter(([, s]) => s.u === user.name).map(([key, s]) => ({
|
||||
id: publicId(key), device: deviceLabel(s.agent), created: s.t, seen: s.seen || s.t,
|
||||
current: key === currentKey, locked: !!s.reader, reader: pick(effective(base, s)),
|
||||
})).sort((a, b) => b.current - a.current || b.seen - a.seen);
|
||||
return { enabled: perSession(base), current: currentKey ? publicId(currentKey) : null,
|
||||
latestFrom: user.readerFrom || null, latest: pick(base), sessions: list };
|
||||
}
|
||||
|
||||
// Settings actions on one of the user's sessions: 'follow' drops its lock, 'latest' makes its set the latest.
|
||||
function act(user, base, sessions, id, action) {
|
||||
const entry = Object.entries(sessions).find(([key, s]) => s.u === user.name && publicId(key) === id);
|
||||
if (!entry) return false;
|
||||
const [key, session] = entry;
|
||||
if (action === 'follow') { delete session.reader; if (user.readerFrom === id) user.readerFrom = null; return true; }
|
||||
if (action === 'latest') {
|
||||
user.settings = { ...(user.settings || {}), ...pick(effective(base, session)) };
|
||||
user.readerFrom = session.reader ? publicId(key) : null;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
module.exports = { READER_KEYS, publicId, pick, effective, applyPatch, noteActive, deviceLabel, overview, act };
|
||||
@@ -35,6 +35,7 @@ async function body(req) {
|
||||
if (url.pathname === '/health') return json(res, 200, { ok: true, status: service.state.status });
|
||||
if (TOKEN && req.headers.authorization !== `Bearer ${TOKEN}`) return json(res, 401, { error: 'Unauthorized' });
|
||||
if (req.method === 'GET' && url.pathname === '/api/state') return json(res, 200, await service.publicState());
|
||||
if (req.method === 'GET' && url.pathname === '/api/revision') return json(res, 200, service.revision());
|
||||
if (req.method !== 'POST') return json(res, 404, { error: 'Not found' });
|
||||
const input = await body(req);
|
||||
if (url.pathname === '/api/queue') return json(res, 200, { item: await service.enqueue(input), state: await service.publicState() });
|
||||
|
||||
+15
-4
@@ -16,6 +16,7 @@ const { writeJsonAtomic } = require('./lib/json-file');
|
||||
const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates');
|
||||
|
||||
const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs'];
|
||||
const INVENTORY_SUMMARY_MS = 15 * 60000; // status summary age limit; decisions still scan (libraryInventory)
|
||||
const FAST_RATE_SETTINGS = {
|
||||
minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000,
|
||||
};
|
||||
@@ -401,7 +402,7 @@ class ScraperService {
|
||||
} catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); }
|
||||
}
|
||||
// The reader reconciles fully when the revision rises without a visible folder change.
|
||||
if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
|
||||
if (written) { this.invalidateInventory(); this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
|
||||
await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written });
|
||||
return { checked, written };
|
||||
}
|
||||
@@ -583,6 +584,15 @@ class ScraperService {
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
inventoryStale() {
|
||||
return this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= INVENTORY_SUMMARY_MS;
|
||||
}
|
||||
|
||||
// What the reader polls every few seconds: no library walk, no event log read.
|
||||
revision() {
|
||||
return { publishedRevision: this.state.publishedRevision || 0, status: this.state.status, paused: this.state.paused };
|
||||
}
|
||||
|
||||
async publicState() {
|
||||
const activeItems = this.state.queue.filter(item => Object.values(this.state.currentIds || {}).includes(item.id));
|
||||
const active = await Promise.all(activeItems.map(async current => ({
|
||||
@@ -593,8 +603,9 @@ class ScraperService {
|
||||
const job = active[0]?.job || null;
|
||||
// Status and watchdog probes must not wait for thousands of bind-mount reads.
|
||||
// Share one background scan, serve the previous summary, and surface its age/error.
|
||||
if (!this.inventoryRefresh && (this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000)
|
||||
&& Date.now() - this.inventoryAttemptedAt >= 10000) {
|
||||
// The scraper is the library's only writer and invalidates the summary on every publish, so the age
|
||||
// limit only catches outside edits. A walk reads every chapter.json: on a NAS that is ~10,000 opens.
|
||||
if (!this.inventoryRefresh && this.inventoryStale() && Date.now() - this.inventoryAttemptedAt >= 10000) {
|
||||
this.refreshInventory().catch(() => {});
|
||||
}
|
||||
const inventory = this.inventoryCache;
|
||||
@@ -609,7 +620,7 @@ class ScraperService {
|
||||
updateChecks: Object.values(this.state.updateChecks || {}),
|
||||
libraryStatus: {
|
||||
ready: !!inventory, refreshing: !!this.inventoryRefresh,
|
||||
stale: this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000,
|
||||
stale: this.inventoryStale(),
|
||||
updated: this.inventoryCachedAt ? new Date(this.inventoryCachedAt).toISOString() : null,
|
||||
error: this.inventoryError,
|
||||
},
|
||||
|
||||
@@ -80,6 +80,23 @@ test('background inventory failures are visible without breaking controls or lau
|
||||
assert.equal(scans, 2);
|
||||
});
|
||||
|
||||
test('status polling walks the library again only after a change or after 15 minutes', async () => {
|
||||
let scans = 0;
|
||||
const service = fixture(async () => { scans++; return inventory('Demo'); });
|
||||
await service.libraryInventory(true);
|
||||
service.inventoryCachedAt = Date.now() - 10 * 60000; service.inventoryAttemptedAt = 0;
|
||||
for (let n = 0; n < 5; n++) await service.publicState();
|
||||
assert.equal(scans, 1);
|
||||
assert.deepEqual(service.revision(), { publishedRevision: 0, status: 'paused', paused: true });
|
||||
assert.equal(scans, 1);
|
||||
service.invalidateInventory();
|
||||
await service.publicState(); await service.inventoryRefresh;
|
||||
assert.equal(scans, 2);
|
||||
service.inventoryCachedAt = Date.now() - 16 * 60000; service.inventoryAttemptedAt = 0;
|
||||
await service.publicState(); await service.inventoryRefresh;
|
||||
assert.equal(scans, 3);
|
||||
});
|
||||
|
||||
test('dashboard polling reuses a recent snapshot while decisions still force a fresh scan', async () => {
|
||||
let scans = 0;
|
||||
const service = fixture(async () => { scans++; return inventory('Demo'); });
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
* SCRAPER_URL=http://127.0.0.1:8091 scraper daemon API
|
||||
* SCRAPER_TOKEN=... shared daemon token (optional on loopback)
|
||||
* BACKUP=/backup folder for built-in DATA backups (unset = backups off)
|
||||
* READ_AHEAD=1 read an opened chapter's pages (then the next chapter's) into the OS cache, 0 = off
|
||||
* PERF_TRACE=1 performance trace to DATA/perf/ (or touch DATA/perf/trace.on at runtime; docs/performance.md)
|
||||
*/
|
||||
const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||
const { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex } = require('./library');
|
||||
@@ -36,6 +38,9 @@ const { chapterDates } = require('./chapter-dates');
|
||||
const { parseSchedule, publicSchedule, loadQuestionBank, createReadingAccess } = require('./reading-access');
|
||||
const { BackupService, BackupError } = require('./backup');
|
||||
const moments = require('./moments');
|
||||
const { Tracer, requestTrace } = require('./perf/trace');
|
||||
const { ReadAhead } = require('./perf/read-ahead');
|
||||
const readerSessions = require('./reader-sessions');
|
||||
|
||||
const PORT = +process.env.PORT || 8080;
|
||||
const LIB = path.resolve(process.env.LIBRARY || 'library');
|
||||
@@ -64,6 +69,9 @@ const MIME = {
|
||||
|
||||
// ---------- storage: one JSON file, written atomically ----------
|
||||
fs.mkdirSync(DATA, { recursive: true });
|
||||
const tracer = new Tracer({ dir: path.join(DATA, 'perf') }).start();
|
||||
const readAhead = process.env.READ_AHEAD === '0' ? null
|
||||
: new ReadAhead({ onBatch: batch => tracer.log({ k: 'ahead', ...batch }) });
|
||||
const catalog = new Catalog(DATA);
|
||||
const greatestCatalog = loadGreatestCatalog();
|
||||
const greatestById = new Map(greatestCatalog.entries.map(entry => [entry.id, entry]));
|
||||
@@ -112,8 +120,10 @@ function checkPw(pw, stored) {
|
||||
}
|
||||
const DUMMY_HASH = hashPw('x');
|
||||
const DEFAULTS = { startMode: 'last', lastMode: 'page', adapt: true, skipExtras: false, skipCovers: false, flowPageGap: true, pageTone: 'original', readingClock: false, readingStory: true,
|
||||
continueRow: true, readDate: 'hour', releaseDate: true, discoveryEnabled: false, discoveryReadingActivity: false, discoveryMature: false, zoom: {}, webhook: '' };
|
||||
const settingsOf = u => ({ ...DEFAULTS, ...u.settings, zoom: { ...(u.settings && u.settings.zoom) } });
|
||||
continueRow: true, readDate: 'hour', releaseDate: true, discoveryEnabled: false, discoveryReadingActivity: false, discoveryMature: false, zoom: {}, webhook: '', readerPerSession: true };
|
||||
const userSettingsOf = u => ({ ...DEFAULTS, ...u.settings, zoom: { ...(u.settings && u.settings.zoom) } });
|
||||
// With a login session, its locked reader settings (reader-sessions.js) win over the user's latest set.
|
||||
const settingsOf = (u, sess) => readerSessions.effective(userSettingsOf(u), sess);
|
||||
|
||||
function createUser(name, pw, admin, replace) {
|
||||
name = String(name || '').trim().toLowerCase(); pw = String(pw || '');
|
||||
@@ -125,13 +135,21 @@ function createUser(name, pw, admin, replace) {
|
||||
save();
|
||||
return db.users[name];
|
||||
}
|
||||
function startSession(res, user) {
|
||||
function startSession(req, res, user) {
|
||||
const token = crypto.randomBytes(32).toString('base64url');
|
||||
db.sessions[sha(token)] = { u: user.name, t: Date.now() };
|
||||
db.sessions[sha(token)] = { u: user.name, t: Date.now(), agent: String(req.headers['user-agent'] || '').slice(0, 300) };
|
||||
save();
|
||||
res.setHeader('Set-Cookie', `sid=${token}; Path=/; HttpOnly; SameSite=Lax; Max-Age=31536000${SECURE_COOKIE ? '; Secure' : ''}`);
|
||||
}
|
||||
const tokenOf = req => { const m = /(?:^|;\s*)sid=([^;]+)/.exec(req.headers.cookie || ''); return m ? m[1] : null; };
|
||||
function sessionKeyOf(req) { const t = tokenOf(req); return t ? sha(t) : null; }
|
||||
// Last-seen time (coarse, to keep db.json writes rare) and device label of a login session.
|
||||
function touchSession(u, sess, key, req) {
|
||||
let changed = false;
|
||||
if (!sess.agent && req.headers['user-agent']) { sess.agent = String(req.headers['user-agent']).slice(0, 300); changed = true; }
|
||||
if (Date.now() - (sess.seen || 0) > 5 * 60e3) { sess.seen = Date.now(); changed = true; }
|
||||
if (changed) save();
|
||||
}
|
||||
function userOf(req) { const t = tokenOf(req), s = t && db.sessions[sha(t)]; return (s && db.users[s.u]) || null; }
|
||||
for (const [k, s] of Object.entries(db.sessions)) if (Date.now() - s.t > 365 * 864e5 || !db.users[s.u]) delete db.sessions[k];
|
||||
|
||||
@@ -175,8 +193,10 @@ async function performScan(mode) {
|
||||
scanState = { at: Date.now(), ms: Date.now() - t0, mode, changedMangas: changed.length,
|
||||
mangas: lib.mangas.length, chapters, pages, stale: false, error: null };
|
||||
console.log(`[scan] ${mode}: ${changed.length} changed manga, ${lib.mangas.length} manga, ${chapters} chapters, ${pages} pages in ${scanState.ms} ms`);
|
||||
tracer.log({ k: 'scan', mode, ms: scanState.ms, changed: changed.length });
|
||||
return scanState;
|
||||
} catch (e) {
|
||||
tracer.log({ k: 'scan', mode, ms: Date.now() - t0, error: e.message });
|
||||
scanState = { ...scanState, at: Date.now(), error: e.message };
|
||||
console.error('[scan] failed:', e.message);
|
||||
return scanState;
|
||||
@@ -253,9 +273,14 @@ async function scraperRequest(endpoint, body) {
|
||||
options.method = 'POST'; headers['content-type'] = 'application/json'; options.body = JSON.stringify(body);
|
||||
}
|
||||
let response;
|
||||
const t0 = performance.now();
|
||||
try { response = await fetch(SCRAPER_URL + endpoint, options); }
|
||||
catch (error) { throw new HttpError(503, `Scraper service unavailable: ${error.message}`); }
|
||||
catch (error) {
|
||||
tracer.log({ k: 'scraper', endpoint, ms: Math.round(performance.now() - t0), error: error.message });
|
||||
throw new HttpError(503, `Scraper service unavailable: ${error.message}`);
|
||||
}
|
||||
const data = await response.json().catch(() => ({}));
|
||||
tracer.log({ k: 'scraper', endpoint, ms: Math.round(performance.now() - t0), status: response.status });
|
||||
if (!response.ok) throw new HttpError(response.status >= 500 ? 502 : response.status, data.error || `Scraper HTTP ${response.status}`);
|
||||
return data;
|
||||
}
|
||||
@@ -433,6 +458,17 @@ function continueOf(m, prog, s) {
|
||||
const next = m.chapters.slice(m.chapters.indexOf(last.ch) + 1).find(ok);
|
||||
return next ? { c: next.id, text: 'Read next · ' + next.label } : { c: last.ch.id, v: last.p.v, text: 'Read again · ' + last.ch.label };
|
||||
}
|
||||
// From the saved page on, then the pages before it; afterwards the next chapter's preferred version.
|
||||
function warmChapter(u, m, ch, v, progress, s) {
|
||||
const from = Math.min(Math.max(0, +(progress && progress.v === v.id && progress.p) || 0), v.pages.length);
|
||||
const pages = v.pages.slice(from).concat(v.pages.slice(0, from)).map(p => path.join(LIB, v.rel, p.f));
|
||||
readAhead.warm(pages, `${m.id}/${ch.id}`, u.name);
|
||||
const next = neighbour(m, ch, 1, s), nextCh = next && m.chapterById.get(next.id);
|
||||
if (nextCh) {
|
||||
const nv = ranked(m, nextCh)[0].v;
|
||||
readAhead.then(nv.pages.map(p => path.join(LIB, nv.rel, p.f)), `${m.id}/${nextCh.id}`, u.name);
|
||||
}
|
||||
}
|
||||
function neighbour(m, ch, step, s) {
|
||||
for (let i = m.chapters.indexOf(ch) + step; i >= 0 && i < m.chapters.length; i += step) {
|
||||
const c = m.chapters[i];
|
||||
@@ -446,11 +482,11 @@ const PUBLIC = new Set(['/api/auth', '/api/login', '/api/signup']);
|
||||
const routes = {
|
||||
'GET /api/health': () => ({ ok: !scanState.error, service: 'reader', uptime: Math.round(process.uptime()), scan: scanState }),
|
||||
'GET /api/collections': ({ u }) => ({ collections: routes['GET /api/library']({ u }).collections }),
|
||||
'GET /api/collection': ({ u, q }) => {
|
||||
'GET /api/collection': ({ u, q, sess }) => {
|
||||
const view = routes['GET /api/library']({ u }).collections.find(c => c.id === q.get('id')) || bad(404, 'Collection not found');
|
||||
const next = view.members.find(m => m.read < m.chapters) || view.members[0];
|
||||
const manga = mangaOf(next.id);
|
||||
return { ...view, continue: { m: next.id, ...continueOf(manga, progressOf(u, manga), settingsOf(u)) },
|
||||
return { ...view, continue: { m: next.id, ...continueOf(manga, progressOf(u, manga), settingsOf(u, sess)) },
|
||||
relations: catalog.state.relations.filter(r => r.state === 'confirmed' && view.members.some(m => m.id === r.from || m.id === r.to)) };
|
||||
},
|
||||
'GET /api/admin/catalog': ({ u }) => { admin(u); return { ...catalog.state, types: TYPES, roles: ROLES, mangas: lib.mangas.map(m => ({ id: m.id, title: m.title })) }; },
|
||||
@@ -498,16 +534,16 @@ const routes = {
|
||||
},
|
||||
'GET /api/auth': () => ({ firstRun: !Object.keys(db.users).length, signup: OPEN_SIGNUP }),
|
||||
|
||||
'POST /api/signup': ({ b, res }) => {
|
||||
'POST /api/signup': ({ req, b, res }) => {
|
||||
const first = !Object.keys(db.users).length;
|
||||
if (!first && !OPEN_SIGNUP) bad(403, 'Sign-up is closed. Ask an admin for an account.');
|
||||
startSession(res, createUser(b.name, b.password, first));
|
||||
startSession(req, res, createUser(b.name, b.password, first));
|
||||
return { ok: true };
|
||||
},
|
||||
'POST /api/login': ({ b, res }) => {
|
||||
'POST /api/login': ({ req, b, res }) => {
|
||||
const user = db.users[String(b.name || '').trim().toLowerCase()], pw = String(b.password || '');
|
||||
if (!checkPw(pw, user ? user.hash : DUMMY_HASH) || !user) bad(403, 'Wrong user name or password.');
|
||||
startSession(res, user);
|
||||
startSession(req, res, user);
|
||||
return { ok: true };
|
||||
},
|
||||
'POST /api/logout': ({ req, res }) => {
|
||||
@@ -516,7 +552,18 @@ const routes = {
|
||||
res.setHeader('Set-Cookie', 'sid=; Path=/; HttpOnly; SameSite=Lax; Max-Age=0');
|
||||
return { ok: true };
|
||||
},
|
||||
'GET /api/me': ({ u }) => ({ name: u.name, admin: !!u.admin, settings: settingsOf(u), unread: unreadOf(u) }),
|
||||
// Every page loads /api/me, so a page view marks its session as the last active one for reader settings.
|
||||
'GET /api/me': ({ u, sess, sk }) => {
|
||||
if (sess && readerSessions.noteActive(u, userSettingsOf(u), sess, sk)) save();
|
||||
return { name: u.name, admin: !!u.admin, settings: settingsOf(u, sess), unread: unreadOf(u) };
|
||||
},
|
||||
'GET /api/reader-sessions': ({ u, sk }) => readerSessions.overview(u, userSettingsOf(u), db.sessions, sk),
|
||||
'POST /api/reader-sessions': ({ u, b, sk }) => {
|
||||
if (!['follow', 'latest'].includes(b.action)) bad(400, 'Unknown action');
|
||||
if (!readerSessions.act(u, userSettingsOf(u), db.sessions, String(b.id || ''), b.action)) bad(404, 'Session not found');
|
||||
save();
|
||||
return readerSessions.overview(u, userSettingsOf(u), db.sessions, sk);
|
||||
},
|
||||
|
||||
'GET /api/reading-schedule': () => publicSchedule(readingSchedule),
|
||||
'GET /api/reading-access': ({ u, q }) => { mangaOf(q.get('m')); return readingAccess.status(u.name, true); },
|
||||
@@ -533,22 +580,23 @@ const routes = {
|
||||
return { ok: true };
|
||||
},
|
||||
|
||||
'POST /api/settings': ({ u, b }) => {
|
||||
'POST /api/settings': ({ u, b, sess, sk }) => {
|
||||
if (typeof b.webhook === 'string' && b.webhook.trim() && !/^https?:\/\/\S+$/i.test(b.webhook.trim())) bad(400, 'The webhook must be an http(s) URL.');
|
||||
const s = (u.settings = u.settings || {});
|
||||
const base = userSettingsOf(u), s = {};
|
||||
if (['last', 'flow', 'page', 'book'].includes(b.startMode)) s.startMode = b.startMode;
|
||||
if (['flow', 'page', 'book'].includes(b.lastMode)) s.lastMode = b.lastMode;
|
||||
if (['original', 'inverted', 'sepia', 'dim', 'extra-dim'].includes(b.pageTone)) s.pageTone = b.pageTone;
|
||||
if (['off', 'date', 'hour'].includes(b.readDate)) s.readDate = b.readDate;
|
||||
for (const k of ['adapt', 'skipExtras', 'skipCovers', 'flowPageGap', 'readingClock', 'readingStory', 'releaseDate', 'continueRow',
|
||||
'discoveryEnabled', 'discoveryReadingActivity', 'discoveryMature']) if (typeof b[k] === 'boolean') s[k] = b[k];
|
||||
'discoveryEnabled', 'discoveryReadingActivity', 'discoveryMature', 'readerPerSession']) if (typeof b[k] === 'boolean') s[k] = b[k];
|
||||
if (b.zoom && typeof b.zoom === 'object') {
|
||||
s.zoom = {};
|
||||
for (const k of ['flow', 'page', 'book', 'strip']) if (+b.zoom[k] > 0) s.zoom[k] = Math.min(8, Math.max(0.1, +b.zoom[k]));
|
||||
}
|
||||
if (typeof b.webhook === 'string') s.webhook = b.webhook.trim().slice(0, 500);
|
||||
readerSessions.applyPatch({ user: u, base, session: sess, key: sk, patch: s });
|
||||
save();
|
||||
return { settings: settingsOf(u) };
|
||||
return { settings: settingsOf(u, sess) };
|
||||
},
|
||||
'POST /api/password': ({ u, b }) => {
|
||||
if (!checkPw(String(b.old || ''), u.hash)) bad(403, 'The current password is wrong.');
|
||||
@@ -676,8 +724,8 @@ const routes = {
|
||||
return { results: await searchGreatestMangaDex(entry) };
|
||||
},
|
||||
|
||||
'GET /api/manga': ({ u, q }) => {
|
||||
const m = mangaOf(q.get('m')), prog = progressOf(u, m), s = settingsOf(u), showStory = s.readingStory;
|
||||
'GET /api/manga': ({ u, q, sess }) => {
|
||||
const m = mangaOf(q.get('m')), prog = progressOf(u, m), s = settingsOf(u, sess), showStory = s.readingStory;
|
||||
const records = mangaReadingOf(u, m);
|
||||
return {
|
||||
chapterDates: { read: s.readDate, release: s.releaseDate },
|
||||
@@ -694,7 +742,7 @@ const routes = {
|
||||
dates: s.releaseDate ? chapterDates(ch.versions) : null,
|
||||
})),
|
||||
reading: showStory ? mangaInsight(records, m.chapters.map(ch => ch.id)) : null,
|
||||
continue: continueOf(m, prog, settingsOf(u)),
|
||||
continue: continueOf(m, prog, s),
|
||||
};
|
||||
},
|
||||
|
||||
@@ -735,15 +783,17 @@ const routes = {
|
||||
return { ok: true, deleted: m.id };
|
||||
},
|
||||
|
||||
'GET /api/chapter': ({ u, q }) => {
|
||||
const m = mangaOf(q.get('m')), ch = chapterOf(m, q.get('c')), s = settingsOf(u), prog = progressOf(u, m);
|
||||
'GET /api/chapter': ({ u, q, sess }) => {
|
||||
const m = mangaOf(q.get('m')), ch = chapterOf(m, q.get('c')), s = settingsOf(u, sess), prog = progressOf(u, m);
|
||||
if (!readingAccess.status(u.name).allowed) bad(423, 'Reading is outside the deployment schedule. Complete the five-question challenge to unlock it.');
|
||||
const v = ch.versions.find(x => x.id === q.get('v')) || ranked(m, ch)[0].v;
|
||||
// Opening a chapter clears its "new chapter" notification.
|
||||
let seen = false;
|
||||
for (const n of db.notes[u.name] || []) if (!n.read && n.m === m.id && n.c === ch.id) seen = n.read = true;
|
||||
if (seen) save();
|
||||
if (readAhead) warmChapter(u, m, ch, v, prog[ch.id], s);
|
||||
return {
|
||||
perf: tracer.on,
|
||||
manga: { id: m.id, title: m.title },
|
||||
chapter: { id: ch.id, label: ch.label, title: ch.title, extra: ch.extra },
|
||||
version: v.id, versions: versionsOf(m, ch, u), direction: v.direction,
|
||||
@@ -875,6 +925,21 @@ const routes = {
|
||||
return { ok: true };
|
||||
},
|
||||
|
||||
// Timings the reader page measured (resource timing, decode, page-turn waits) while the trace is on.
|
||||
'POST /api/perf': ({ u, b }) => {
|
||||
if (!tracer.on) return { on: false };
|
||||
const events = Array.isArray(b.events) ? b.events.slice(0, 300) : [];
|
||||
for (const e of events) if (e && typeof e === 'object' && !Array.isArray(e)) {
|
||||
const clean = {};
|
||||
for (const [key, value] of Object.entries(e).slice(0, 30)) {
|
||||
if (key === 't' || key === 'k') continue; // the server's clock and kind
|
||||
if (typeof value === 'number' && Number.isFinite(value)) clean[key] = Math.round(value * 10) / 10;
|
||||
else if (typeof value === 'string' || typeof value === 'boolean') clean[key] = typeof value === 'string' ? value.slice(0, 300) : value;
|
||||
}
|
||||
tracer.log({ ...clean, k: 'client', user: u.name, page: b.page ? String(b.page).slice(0, 40) : undefined });
|
||||
}
|
||||
return { on: true };
|
||||
},
|
||||
'GET /api/notifications': ({ u }) => ({ items: (db.notes[u.name] || []).slice().reverse() }),
|
||||
'POST /api/notifications/read': ({ u, b }) => {
|
||||
for (const n of db.notes[u.name] || []) if (b.all || n.id === b.id) n.read = true;
|
||||
@@ -1060,23 +1125,32 @@ async function handleApi(req, res, u, route, q) {
|
||||
if (!/^application\/json/.test(req.headers['content-type'] || '') || req.headers['sec-fetch-site'] === 'cross-site') bad(403, 'Bad request origin');
|
||||
b = (await readBody(req)) || {};
|
||||
}
|
||||
sendJson(res, 200, await route({ req, res, u, b, q }));
|
||||
const sk = u && sessionKeyOf(req), sess = sk && db.sessions[sk];
|
||||
if (sess) touchSession(u, sess, sk, req);
|
||||
sendJson(res, 200, await route({ req, res, u, b, q, sess, sk }));
|
||||
}
|
||||
function serveFile(req, res, root, rel, cacheControl, only) {
|
||||
function serveFile(req, res, root, rel, cacheControl, only, trace) {
|
||||
let file;
|
||||
try { file = path.resolve(root, decodeURIComponent(rel)); } catch { return sendJson(res, 400, { error: 'Bad path' }); }
|
||||
if (!file.startsWith(root + path.sep) || (only && !only.test(file))) return sendJson(res, 404, { error: 'Not found' });
|
||||
fs.stat(file, (err, st) => {
|
||||
if (trace) trace.mark('stat');
|
||||
if (err || !st.isFile()) return sendJson(res, 404, { error: 'Not found' });
|
||||
const etag = `"${st.size.toString(36)}-${Math.round(st.mtimeMs).toString(36)}"`;
|
||||
if (req.headers['if-none-match'] === etag) { res.writeHead(304, { ETag: etag, 'Cache-Control': cacheControl }); return res.end(); }
|
||||
res.writeHead(200, { 'Content-Type': MIME[path.extname(file).toLowerCase()] || 'application/octet-stream', 'Content-Length': st.size, ETag: etag, 'Cache-Control': cacheControl });
|
||||
const headers = { 'Content-Type': MIME[path.extname(file).toLowerCase()] || 'application/octet-stream', 'Content-Length': st.size, ETag: etag, 'Cache-Control': cacheControl };
|
||||
// The browser's resource timing shows this, so a slow page can be matched to the server's share.
|
||||
if (trace && trace.on) headers['Server-Timing'] = `stat;dur=${trace.elapsed()}`;
|
||||
res.writeHead(200, headers);
|
||||
if (req.method === 'HEAD') return res.end();
|
||||
fs.createReadStream(file).on('error', () => res.destroy()).pipe(res);
|
||||
const stream = fs.createReadStream(file).on('error', () => res.destroy());
|
||||
if (trace) trace.stream(stream);
|
||||
stream.pipe(res);
|
||||
});
|
||||
}
|
||||
|
||||
http.createServer(async (req, res) => {
|
||||
const trace = requestTrace(tracer, req, res);
|
||||
try {
|
||||
const { pathname, searchParams } = new URL(req.url, 'http://localhost');
|
||||
const u = userOf(req);
|
||||
@@ -1124,9 +1198,10 @@ http.createServer(async (req, res) => {
|
||||
const parts = relative.split(path.sep);
|
||||
if (parts.length >= 4 && !readingAccess.status(u.name).allowed)
|
||||
bad(423, 'Reading is outside the deployment schedule.');
|
||||
return serveFile(req, res, LIB, pathname.slice(5), readingSchedule && parts.length >= 4 ? 'private, no-store' : 'private, max-age=86400', IMG);
|
||||
trace.mark('access');
|
||||
return serveFile(req, res, LIB, pathname.slice(5), readingSchedule && parts.length >= 4 ? 'private, no-store' : 'private, max-age=86400', IMG, trace);
|
||||
}
|
||||
serveFile(req, res, PUB, pathname === '/' ? 'index.html' : pathname.slice(1), 'no-cache');
|
||||
serveFile(req, res, PUB, pathname === '/' ? 'index.html' : pathname.slice(1), 'no-cache', null, trace);
|
||||
} catch (e) {
|
||||
const expected = e instanceof HttpError || e instanceof CatalogError || e instanceof moments.MomentError;
|
||||
if (!expected) console.error(e);
|
||||
@@ -1134,6 +1209,13 @@ http.createServer(async (req, res) => {
|
||||
}
|
||||
}).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`));
|
||||
|
||||
// The trace's storage probe reads a random page of the library (rarely in the OS cache).
|
||||
tracer.setProbeSource(() => {
|
||||
const m = lib.mangas[Math.floor(Math.random() * lib.mangas.length)];
|
||||
const ch = m && m.chapters[Math.floor(Math.random() * m.chapters.length)], v = ch && ch.versions[0];
|
||||
const p = v && v.pages[Math.floor(Math.random() * v.pages.length)];
|
||||
return p ? path.join(LIB, v.rel, p.f) : null;
|
||||
});
|
||||
if (lib.mangas.length) refreshChanged();
|
||||
else rescan();
|
||||
if (SCAN_MINUTES > 0) setInterval(() => refreshChanged(), SCAN_MINUTES * 60000).unref();
|
||||
@@ -1161,10 +1243,13 @@ if (FULL_SCAN_MINUTES > 0) setInterval(() => {
|
||||
|
||||
// A completed scraper publish increments a durable revision. Polling it here keeps the
|
||||
// reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.
|
||||
let scraperRevision = null;
|
||||
let scraperRevision = null, revisionRoute = true;
|
||||
async function pollScraper() {
|
||||
try {
|
||||
const state = await scraperRequest('/api/state');
|
||||
// /api/revision is a few bytes; the full state is ~100 KB and makes the scraper summarise the library.
|
||||
let state;
|
||||
if (revisionRoute) state = await scraperRequest('/api/revision').catch(error => { if (error.code !== 404) throw error; revisionRoute = false; });
|
||||
if (!revisionRoute) state = await scraperRequest('/api/state'); // a scraper from before /api/revision
|
||||
const revision = Number(state.publishedRevision || 0);
|
||||
if (scraperRevision === null || revision > scraperRevision) {
|
||||
const checked = await refreshChanged();
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
'use strict';
|
||||
// The image copies top-level modules by name; a new module missing there crashes the reader at start.
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('fs'), path = require('path');
|
||||
const root = path.resolve(__dirname, '..');
|
||||
const allowedNames = () => new Set(fs.readFileSync(path.join(root, '.dockerignore'), 'utf8').split(/\r?\n/).filter(line => line.startsWith('!')).map(line => line.slice(1)));
|
||||
|
||||
test('every top-level module the reader requires is copied into the image', () => {
|
||||
const dockerfile = fs.readFileSync(path.join(root, 'Dockerfile'), 'utf8');
|
||||
const copied = new Set(dockerfile.split('\n').filter(line => line.startsWith('COPY ')).flatMap(line => line.split(/\s+/).slice(1, -1)));
|
||||
const pending = ['server.js'], seen = new Set();
|
||||
while (pending.length) {
|
||||
const file = pending.pop();
|
||||
if (seen.has(file)) continue;
|
||||
seen.add(file);
|
||||
const source = fs.readFileSync(path.join(root, file), 'utf8');
|
||||
for (const [, dir] of source.matchAll(/require\('\.\/([\w.-]+)\/[^']+'\)/g)) {
|
||||
assert.ok(copied.has(dir), `${dir}/ is not copied`);
|
||||
assert.ok(allowedNames().has(dir + '/'), `${dir}/ is not allowed through .dockerignore`);
|
||||
}
|
||||
for (const [, name] of source.matchAll(/require\('\.\/([\w.-]+?)(?:\.js)?'\)/g)) {
|
||||
// A folder (e.g. ./greatest) is copied as a whole; only single files need their own name in COPY.
|
||||
if (fs.existsSync(path.join(root, name)) && fs.statSync(path.join(root, name)).isDirectory()) assert.ok(copied.has(name), `${name}/ is not copied`);
|
||||
else pending.push(name + '.js');
|
||||
}
|
||||
}
|
||||
// .dockerignore is an allowlist too: a file not let through never reaches COPY.
|
||||
const allowed = allowedNames();
|
||||
for (const file of seen) {
|
||||
assert.ok(copied.has(file), `${file} is required but not copied by the Dockerfile`);
|
||||
assert.ok(allowed.has(file), `${file} is required but not allowed through .dockerignore`);
|
||||
}
|
||||
});
|
||||
@@ -143,7 +143,8 @@ test('moments are private, validated, editable and survive page renumbering as "
|
||||
assert.equal((await post(alice, '/api/moments/delete', { id: spread.id })).status, 200);
|
||||
assert.equal((await get(alice, '/api/moments?m=Moon%20Garden')).moments.length, 1);
|
||||
} finally {
|
||||
if (child) child.kill();
|
||||
// The server writes db.json on SIGTERM; removing the folder before it exits races that write.
|
||||
if (child && child.exitCode === null) await new Promise(resolve => { child.once('exit', resolve); child.kill(); });
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('fs'), os = require('os'), path = require('path'), http = require('http');
|
||||
const { Tracer, requestTrace } = require('../perf/trace');
|
||||
const { ReadAhead } = require('../perf/read-ahead');
|
||||
const { analyse } = require('../tools/perf-report');
|
||||
|
||||
const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'circus-perf-'));
|
||||
const lines = dir => fs.readdirSync(dir).filter(n => n.endsWith('.jsonl'))
|
||||
.flatMap(n => fs.readFileSync(path.join(dir, n), 'utf8').trim().split('\n')).filter(Boolean).map(JSON.parse);
|
||||
|
||||
test('tracer is off without the flag file and writes nothing', () => {
|
||||
const dir = tmp(), tracer = new Tracer({ dir, env: {} });
|
||||
tracer.check(); tracer.log({ k: 'x' }); tracer.flush();
|
||||
assert.equal(tracer.on, false);
|
||||
assert.deepEqual(fs.readdirSync(dir), []);
|
||||
});
|
||||
|
||||
test('the flag file switches the trace on and off at runtime', () => {
|
||||
const dir = tmp(), tracer = new Tracer({ dir, env: {} });
|
||||
fs.writeFileSync(path.join(dir, 'trace.on'), '');
|
||||
tracer.check(); tracer.log({ k: 'x', a: 1 });
|
||||
fs.rmSync(path.join(dir, 'trace.on'));
|
||||
tracer.check(); tracer.log({ k: 'x', a: 2 }); tracer.flush();
|
||||
const events = lines(dir);
|
||||
assert.deepEqual(events.map(e => e.k), ['trace', 'x', 'trace']);
|
||||
assert.equal(events[1].a, 1);
|
||||
assert.equal(events[2].on, false);
|
||||
assert.ok(events.every(e => typeof e.t === 'number'));
|
||||
});
|
||||
|
||||
test('a served file is traced with storage and network phases', async () => {
|
||||
const dir = tmp(), tracer = new Tracer({ dir, env: { PERF_TRACE: '1' } });
|
||||
tracer.check();
|
||||
const file = path.join(dir, 'page.jpg');
|
||||
fs.writeFileSync(file, Buffer.alloc(3 * 1024 * 1024, 7));
|
||||
const server = http.createServer((req, res) => {
|
||||
const trace = requestTrace(tracer, req, res);
|
||||
fs.stat(file, (err, st) => {
|
||||
trace.mark('stat');
|
||||
res.writeHead(200, { 'Content-Length': st.size });
|
||||
const stream = fs.createReadStream(file);
|
||||
trace.stream(stream);
|
||||
stream.pipe(res);
|
||||
});
|
||||
});
|
||||
await new Promise(r => server.listen(0, r));
|
||||
const { port } = server.address();
|
||||
const r = await fetch(`http://127.0.0.1:${port}/lib/M/B/C/page.jpg`);
|
||||
assert.equal((await r.arrayBuffer()).byteLength, 3 * 1024 * 1024);
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
server.close(); tracer.stopSampler(); tracer.flush();
|
||||
const req = lines(dir).find(e => e.k === 'req');
|
||||
assert.equal(req.status, 200);
|
||||
assert.equal(req.bytes, 3 * 1024 * 1024);
|
||||
assert.equal(req.path, '/lib/M/B/C/page.jpg');
|
||||
for (const field of ['stat', 'open', 'first', 'readEnd', 'disk', 'netWait', 'ms']) assert.equal(typeof req[field], 'number', field);
|
||||
assert.ok(req.ms >= req.readEnd && req.readEnd >= req.first && req.first >= req.open);
|
||||
});
|
||||
|
||||
test('read-ahead reads files in order and a newer chapter replaces the queued one', async () => {
|
||||
const dir = tmp(), files = [];
|
||||
for (let i = 0; i < 6; i++) { const f = path.join(dir, `p${i}.jpg`); fs.writeFileSync(f, Buffer.alloc(1000 + i)); files.push(f); }
|
||||
const batches = [];
|
||||
const ahead = new ReadAhead({ concurrency: 1, onBatch: b => batches.push(b) });
|
||||
const done = new Promise(resolve => { const check = () => (batches.some(b => b.tag === 'B' && !b.replaced) ? resolve() : setTimeout(check, 5)); check(); });
|
||||
ahead.warm(files.slice(0, 4), 'A');
|
||||
ahead.warm(files.slice(3), 'B');
|
||||
await done;
|
||||
assert.equal(batches[0].tag, 'A'); assert.equal(batches[0].replaced, true);
|
||||
const b = batches.find(x => x.tag === 'B');
|
||||
assert.equal(b.done, 3); assert.equal(b.bytes, 1003 + 1004 + 1005);
|
||||
// Files read recently are skipped the next time.
|
||||
batches.length = 0;
|
||||
ahead.warm(files.slice(4), 'C');
|
||||
assert.deepEqual(batches.map(x => [x.tag, x.files, x.skipped]), [['C', 0, 2]]);
|
||||
});
|
||||
|
||||
test('read-ahead keeps one queue per reader and serves them in turn', async () => {
|
||||
const dir = tmp(), mk = n => { const f = path.join(dir, n); fs.writeFileSync(f, n); return f; };
|
||||
const order = [], batches = [];
|
||||
const ahead = new ReadAhead({ concurrency: 1, onBatch: b => batches.push(b) });
|
||||
const read = ahead.readFile.bind(ahead);
|
||||
ahead.readFile = f => { order.push(path.basename(f)); return read(f); };
|
||||
ahead.warm([mk('a1'), mk('a2')], 'A', 'ann');
|
||||
ahead.warm([mk('b1'), mk('b2')], 'B', 'bob');
|
||||
await new Promise(r => { const check = () => (batches.length === 2 ? r() : setTimeout(check, 5)); check(); });
|
||||
assert.deepEqual(order, ['a1', 'b1', 'a2', 'b2']);
|
||||
assert.ok(batches.every(b => !b.replaced && b.done === 2));
|
||||
});
|
||||
|
||||
test('read-ahead runs the follow-up batch after the current one', async () => {
|
||||
const dir = tmp(), a = path.join(dir, 'a.jpg'), b = path.join(dir, 'b.jpg');
|
||||
fs.writeFileSync(a, 'a'); fs.writeFileSync(b, 'bb');
|
||||
const tags = [];
|
||||
const ahead = new ReadAhead({ onBatch: x => tags.push(x.tag) });
|
||||
ahead.warm([a], 'now'); ahead.then([b], 'next');
|
||||
await new Promise(r => { const check = () => (tags.length === 2 ? r() : setTimeout(check, 5)); check(); });
|
||||
assert.deepEqual(tags, ['now', 'next']);
|
||||
});
|
||||
|
||||
test('the report blames the storage for slow first reads', () => {
|
||||
const t = Date.parse('2026-10-29T20:00:00Z');
|
||||
const events = [
|
||||
{ t, k: 'req', m: 'GET', path: '/lib/M/B/C/1.jpg', status: 200, bytes: 300000, stat: 2, open: 3, first: 1900, readEnd: 1950, disk: 1947, netWait: 0, ms: 1955, conn: 1 },
|
||||
{ t: t + 1000, k: 'req', m: 'GET', path: '/lib/M/B/C/2.jpg', status: 200, bytes: 300000, stat: 1, open: 2, first: 5, readEnd: 20, disk: 18, netWait: 0, ms: 22, conn: 2 },
|
||||
{ t: t + 2000, k: 'client', e: 'res', url: '/lib/M/B/C/1.jpg', total: 2100, ttfb: 30, download: 2000, queue: 1, connect: 0, dns: 0 },
|
||||
{ t: t + 3000, k: 'client', e: 'view', wait: 2050, preloaded: false },
|
||||
{ t: t + 4000, k: 'nas', firstMs: 2500, ms: 2600, bytes: 1 },
|
||||
];
|
||||
const o = analyse(events, { slow: 500 });
|
||||
assert.equal(o.slowServer.count, 1);
|
||||
assert.deepEqual(o.slowServer.by, { firstRead: 1 });
|
||||
assert.deepEqual(o.slowBrowser.by, { server: 1 });
|
||||
assert.equal(o.pageTurns.over1s, 1);
|
||||
assert.match(o.verdict.join('\n'), /first read from storage/);
|
||||
});
|
||||
@@ -0,0 +1,106 @@
|
||||
'use strict';
|
||||
// Reader settings per login session: a session follows the latest set until it changes one, then keeps its own.
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('fs'), os = require('os'), path = require('path'), net = require('net');
|
||||
const { spawn } = require('child_process');
|
||||
const readerSessions = require('../reader-sessions');
|
||||
|
||||
async function freePort() {
|
||||
const listener = net.createServer();
|
||||
await new Promise(resolve => listener.listen(0, '127.0.0.1', resolve));
|
||||
const port = listener.address().port;
|
||||
await new Promise(resolve => listener.close(resolve));
|
||||
return port;
|
||||
}
|
||||
|
||||
test('device labels name browser and system', () => {
|
||||
assert.equal(readerSessions.deviceLabel('Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Version/17.0 Mobile/15E148 Safari/604.1'), 'Safari on iPhone');
|
||||
assert.equal(readerSessions.deviceLabel('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/130.0 Safari/537.36 Edg/130.0'), 'Edge on Windows');
|
||||
assert.equal(readerSessions.deviceLabel(''), 'Unknown device');
|
||||
});
|
||||
|
||||
test('sessions lock their reader settings on the first change, new ones copy the last active one', { timeout: 20000 }, async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-reader-sessions-'));
|
||||
let child;
|
||||
try {
|
||||
const library = path.join(root, 'library'), data = path.join(root, 'data');
|
||||
fs.mkdirSync(library); fs.mkdirSync(data);
|
||||
const port = await freePort(), output = [];
|
||||
child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true,
|
||||
env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0',
|
||||
SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '',
|
||||
GREATEST_COVER_WARM: '0' } });
|
||||
child.stdout.on('data', d => output.push(String(d)));
|
||||
child.stderr.on('data', d => output.push(String(d)));
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
let ready = false;
|
||||
for (let n = 0; n < 100 && !ready; n++) {
|
||||
ready = await fetch(base + '/health').then(r => r.ok).catch(() => false);
|
||||
if (!ready) await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
assert.ok(ready, output.join(''));
|
||||
|
||||
const json = async r => { const body = await r.json(); assert.equal(r.status, 200, JSON.stringify(body)); return body; };
|
||||
const post = (cookie, url, body) => fetch(base + url, { method: 'POST',
|
||||
headers: { 'content-type': 'application/json', 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) Firefox/130.0', ...(cookie ? { cookie } : {}) },
|
||||
body: JSON.stringify(body) }).then(async r => (url.endsWith('login') || url.endsWith('signup') ? r : json(r)));
|
||||
const me = async cookie => (await json(await fetch(base + '/api/me', { headers: { cookie } }))).settings;
|
||||
const overview = cookie => fetch(base + '/api/reader-sessions', { headers: { cookie } }).then(json);
|
||||
const login = async () => (await post(null, '/api/login', { name: 'reader', password: 'reader-password' })).headers.get('set-cookie').split(';')[0];
|
||||
|
||||
const a = (await post(null, '/api/signup', { name: 'reader', password: 'reader-password' })).headers.get('set-cookie').split(';')[0];
|
||||
// First session: defaults, following the latest set.
|
||||
assert.equal((await me(a)).lastMode, 'page');
|
||||
assert.equal((await overview(a)).sessions[0].locked, false);
|
||||
|
||||
// Session B exists before A changes anything: it follows A's change.
|
||||
const b = await login();
|
||||
await post(a, '/api/settings', { lastMode: 'flow', zoom: { flow: 1.5 } });
|
||||
assert.equal((await me(b)).lastMode, 'flow');
|
||||
assert.equal((await me(b)).zoom.flow, 1.5);
|
||||
|
||||
// B changes its own: B locks, A keeps its own set.
|
||||
await post(b, '/api/settings', { pageTone: 'sepia' });
|
||||
assert.equal((await me(b)).pageTone, 'sepia');
|
||||
assert.equal((await me(b)).lastMode, 'flow');
|
||||
assert.equal((await me(a)).pageTone, 'original');
|
||||
|
||||
// A non-reader setting is shared and does not lock anything.
|
||||
const c = await login();
|
||||
await post(c, '/api/settings', { continueRow: false });
|
||||
assert.equal((await me(a)).continueRow, false);
|
||||
assert.equal((await overview(c)).sessions.find(s => s.current).locked, false);
|
||||
// C follows the last active session, which is A (it loaded /api/me last).
|
||||
assert.equal((await me(c)).pageTone, 'original');
|
||||
await me(b);
|
||||
assert.equal((await me(c)).pageTone, 'sepia');
|
||||
|
||||
// Overview: three sessions, labels, intent.
|
||||
const o = await overview(a);
|
||||
assert.equal(o.enabled, true);
|
||||
assert.equal(o.sessions.length, 3);
|
||||
assert.equal(o.sessions[0].current, true);
|
||||
assert.equal(o.sessions[0].device, 'Firefox on Linux');
|
||||
assert.deepEqual(o.sessions.map(s => s.locked).sort(), [false, true, true]);
|
||||
|
||||
// "Follow latest again" drops A's lock; "Use for new sessions" makes a set the latest.
|
||||
const idA = o.sessions[0].id, idB = o.sessions.find(s => !s.current && s.locked).id;
|
||||
await post(a, '/api/reader-sessions', { id: idA, action: 'follow' });
|
||||
assert.equal((await me(a)).pageTone, 'sepia');
|
||||
assert.equal((await fetch(base + '/api/reader-sessions', { method: 'POST', headers: { 'content-type': 'application/json', cookie: a },
|
||||
body: JSON.stringify({ id: 'nope', action: 'follow' }) })).status, 404);
|
||||
assert.equal((await post(a, '/api/reader-sessions', { id: idB, action: 'latest' })).latestFrom, idB);
|
||||
|
||||
// Off: one shared set; a change applies everywhere and locks nothing. On again: kept locks return.
|
||||
await post(b, '/api/settings', { readerPerSession: false });
|
||||
await post(a, '/api/settings', { pageTone: 'dim' });
|
||||
assert.equal((await me(b)).pageTone, 'dim');
|
||||
assert.equal((await me(c)).pageTone, 'dim');
|
||||
await post(a, '/api/settings', { readerPerSession: true });
|
||||
assert.equal((await me(b)).pageTone, 'sepia');
|
||||
assert.equal((await me(a)).pageTone, 'sepia');
|
||||
} finally {
|
||||
if (child && child.exitCode === null) { child.kill(); await new Promise(resolve => child.once('exit', resolve)); }
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
/*
|
||||
* Load test for the reader: N simulated readers each open random chapters and fetch the pages one after
|
||||
* another with a think time, the way a person paging through a chapter does, plus a small look-ahead.
|
||||
* Measures time to first byte and total per page from this machine. Read-only apart from the login; it
|
||||
* never writes progress. Run it with the server's trace on to see the server's share (docs/performance.md).
|
||||
*
|
||||
* node tools/perf-bench.js --url http://host:8080 --user bench --password-env BENCH_PASSWORD \
|
||||
* [--readers 4] [--chapters 3] [--pages 40] [--think 400] [--ahead 2] [--manga Berserk] [--json]
|
||||
* --cookie 'sid=…' instead of user/password. --chapter-only skips the page fetches (API timings only).
|
||||
*/
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { url: 'http://localhost:8080', readers: 4, chapters: 3, pages: 40, think: 400, ahead: 2, manga: '', json: false };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const key = argv[i].replace(/^--/, ''), next = () => argv[++i];
|
||||
if (['readers', 'chapters', 'pages', 'think', 'ahead'].includes(key)) o[key] = +next();
|
||||
else if (key === 'password-env') o.password = process.env[next()] || '';
|
||||
else if (['url', 'user', 'cookie', 'manga'].includes(key)) o[key] = next();
|
||||
else if (key === 'json' || key === 'chapter-only') o[key.replace('-o', 'O')] = true;
|
||||
else if (key === 'h' || key === 'help') o.help = true;
|
||||
}
|
||||
o.url = o.url.replace(/\/$/, '');
|
||||
return o;
|
||||
}
|
||||
|
||||
async function login(o) {
|
||||
if (o.cookie) return o.cookie;
|
||||
const r = await fetch(o.url + '/api/login', { method: 'POST', headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ name: o.user, password: o.password }) });
|
||||
if (!r.ok) throw new Error(`login failed: HTTP ${r.status} ${await r.text()}`);
|
||||
const cookie = (r.headers.get('set-cookie') || '').split(';')[0];
|
||||
if (!cookie) throw new Error('login returned no session cookie');
|
||||
return cookie;
|
||||
}
|
||||
|
||||
async function timed(url, cookie) {
|
||||
const t0 = performance.now();
|
||||
const r = await fetch(url, { headers: { cookie } });
|
||||
const ttfb = performance.now() - t0;
|
||||
let bytes = 0;
|
||||
for await (const chunk of r.body) bytes += chunk.length;
|
||||
return { status: r.status, ttfb, ms: performance.now() - t0, bytes, server: r.headers.get('server-timing') || '' };
|
||||
}
|
||||
const json = async (o, cookie, path) => {
|
||||
const t0 = performance.now(), r = await fetch(o.url + path, { headers: { cookie } });
|
||||
if (!r.ok) throw new Error(`${path}: HTTP ${r.status}`);
|
||||
return { data: await r.json(), ms: performance.now() - t0 };
|
||||
};
|
||||
const pick = list => list[Math.floor(Math.random() * list.length)];
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
async function reader(o, cookie, mangas, out, id) {
|
||||
for (let c = 0; c < o.chapters; c++) {
|
||||
const m = pick(mangas);
|
||||
const manga = await json(o, cookie, '/api/manga?m=' + encodeURIComponent(m.id));
|
||||
out.api.push({ path: '/api/manga', ms: manga.ms });
|
||||
const ch = pick(manga.data.chapters);
|
||||
const chapter = await json(o, cookie, `/api/chapter?m=${encodeURIComponent(m.id)}&c=${encodeURIComponent(ch.id)}`);
|
||||
out.api.push({ path: '/api/chapter', ms: chapter.ms });
|
||||
if (o.chapterOnly) continue;
|
||||
const pages = chapter.data.pages.slice(0, o.pages);
|
||||
// The browser keeps `ahead` pages in flight while the reader looks at the current one.
|
||||
const inflight = new Map();
|
||||
const start = k => { if (k < pages.length && !inflight.has(k)) inflight.set(k, timed(o.url + '/' + pages[k].u, cookie)); };
|
||||
for (let k = 0; k < pages.length; k++) {
|
||||
const waitFrom = performance.now();
|
||||
start(k);
|
||||
for (let a = 1; a <= o.ahead; a++) start(k + a);
|
||||
const r = await inflight.get(k);
|
||||
out.pages.push({ reader: id, manga: m.id, chapter: ch.id, page: k, wait: performance.now() - waitFrom, ...r });
|
||||
await sleep(o.think * (0.5 + Math.random()));
|
||||
}
|
||||
await Promise.all(inflight.values());
|
||||
}
|
||||
}
|
||||
|
||||
const pct = (v, p) => { const s = v.slice().sort((a, b) => a - b); return s.length ? Math.round(s[Math.min(s.length - 1, Math.floor(p / 100 * s.length))]) : null; };
|
||||
const sum = v => ({ n: v.length, p50: pct(v, 50), p90: pct(v, 90), p99: pct(v, 99), max: v.length ? Math.round(Math.max(...v)) : null });
|
||||
|
||||
async function main() {
|
||||
const o = parseArgs(process.argv.slice(2));
|
||||
if (o.help || (!o.cookie && !o.user)) {
|
||||
console.log('usage: node tools/perf-bench.js --url URL (--user NAME --password-env VAR | --cookie sid=…) [--readers N] [--chapters N] [--pages N] [--think ms] [--ahead N] [--manga text] [--chapter-only] [--json]');
|
||||
process.exit(o.help ? 0 : 2);
|
||||
}
|
||||
const cookie = await login(o);
|
||||
let mangas = (await json(o, cookie, '/api/library')).data.mangas;
|
||||
if (o.manga) mangas = mangas.filter(m => m.title.toLowerCase().includes(o.manga.toLowerCase()));
|
||||
if (!mangas.length) throw new Error('no manga to read');
|
||||
const out = { pages: [], api: [] }, t0 = Date.now();
|
||||
await Promise.all(Array.from({ length: o.readers }, (_, i) => reader(o, cookie, mangas, out, i)));
|
||||
const ok = out.pages.filter(p => p.status === 200), mb = ok.reduce((a, p) => a + p.bytes, 0) / 1048576;
|
||||
const result = {
|
||||
readers: o.readers, seconds: Math.round((Date.now() - t0) / 1000), pages: out.pages.length, errors: out.pages.length - ok.length,
|
||||
MB: Math.round(mb), wait: sum(ok.map(p => p.wait)), ttfb: sum(ok.map(p => p.ttfb)), total: sum(ok.map(p => p.ms)),
|
||||
waitedOver300: ok.filter(p => p.wait >= 300).length, waitedOver1s: ok.filter(p => p.wait >= 1000).length,
|
||||
api: Object.fromEntries([...new Set(out.api.map(a => a.path))].map(p => [p, sum(out.api.filter(a => a.path === p).map(a => a.ms))])),
|
||||
worst: ok.sort((a, b) => b.wait - a.wait).slice(0, 8).map(p => ({ wait: Math.round(p.wait), ttfb: Math.round(p.ttfb), ms: Math.round(p.ms), kb: Math.round(p.bytes / 1024), at: `${p.manga} ${p.chapter} p${p.page}` })),
|
||||
};
|
||||
if (o.json) return console.log(JSON.stringify(result, null, 2));
|
||||
console.log(`${result.readers} readers, ${result.pages} pages (${result.MB} MB) in ${result.seconds} s, ${result.errors} errors`);
|
||||
for (const k of ['wait', 'ttfb', 'total']) console.log(` ${k.padEnd(6)} ms ${JSON.stringify(result[k])}`);
|
||||
console.log(` page waits >= 300 ms: ${result.waitedOver300}, >= 1 s: ${result.waitedOver1s} (wait = time the reader looked at an empty page)`);
|
||||
for (const [p, s] of Object.entries(result.api)) console.log(` ${p} ${JSON.stringify(s)}`);
|
||||
for (const w of result.worst) console.log(` wait ${w.wait} ms (ttfb ${w.ttfb}, total ${w.ms}, ${w.kb} KB) ${w.at}`);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error(e.message); process.exit(1); });
|
||||
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
/*
|
||||
* Summarise a performance trace (docs/performance.md) and say where slow page loads spend their time.
|
||||
*
|
||||
* node tools/perf-report.js DATA/perf all trace files in the folder
|
||||
* node tools/perf-report.js trace-2026-10-29.jsonl --since 2026-10-29T18:00 --until … --slow 500 --json
|
||||
*
|
||||
* Plain text by default (readable by people and agents); --json prints the numbers only.
|
||||
*/
|
||||
const fs = require('fs'), path = require('path');
|
||||
|
||||
function parseArgs(argv) {
|
||||
const opts = { files: [], since: null, until: null, slow: 500, json: false };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--since') opts.since = Date.parse(argv[++i]);
|
||||
else if (a === '--until') opts.until = Date.parse(argv[++i]);
|
||||
else if (a === '--slow') opts.slow = +argv[++i];
|
||||
else if (a === '--json') opts.json = true;
|
||||
else if (a === '-h' || a === '--help') opts.help = true;
|
||||
else opts.files.push(a);
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
function readEvents(inputs, { since, until }) {
|
||||
const files = inputs.flatMap(p => fs.statSync(p).isDirectory()
|
||||
? fs.readdirSync(p).filter(n => /^trace-.*\.jsonl$/.test(n)).sort().map(n => path.join(p, n)) : [p]);
|
||||
const events = [];
|
||||
for (const file of files) for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
if (!line) continue;
|
||||
let e; try { e = JSON.parse(line); } catch { continue; }
|
||||
if ((since && e.t < since) || (until && e.t > until)) continue;
|
||||
events.push(e);
|
||||
}
|
||||
return events.sort((a, b) => a.t - b.t);
|
||||
}
|
||||
|
||||
const pct = (values, p) => {
|
||||
const v = values.filter(x => typeof x === 'number').sort((a, b) => a - b);
|
||||
if (!v.length) return null;
|
||||
return v[Math.min(v.length - 1, Math.floor((p / 100) * v.length))];
|
||||
};
|
||||
const stats = values => {
|
||||
const v = values.filter(x => typeof x === 'number');
|
||||
return v.length ? { n: v.length, p50: pct(v, 50), p90: pct(v, 90), p99: pct(v, 99), max: Math.max(...v) } : { n: 0 };
|
||||
};
|
||||
const isPageImage = e => e.k === 'req' && e.path.startsWith('/lib/') && e.path.split('/').length >= 6;
|
||||
|
||||
// Where one served image spent its time on the server.
|
||||
function serverPhases(e) {
|
||||
const stat = e.stat ?? 0, open = e.open != null ? e.open - stat : 0, firstRead = e.first != null && e.open != null ? e.first - e.open : 0;
|
||||
const rest = Math.max(0, (e.disk ?? 0) - firstRead), net = e.netWait ?? 0;
|
||||
const tail = e.readEnd != null ? Math.max(0, e.ms - e.readEnd) : 0;
|
||||
const r = v => Math.round(v * 10) / 10;
|
||||
return { stat: r(stat), open: r(open), firstRead: r(firstRead), readRest: r(rest), netWait: r(net), tail: r(tail) };
|
||||
}
|
||||
const dominant = phases => Object.entries(phases).sort((a, b) => b[1] - a[1])[0][0];
|
||||
|
||||
function analyse(events, { slow }) {
|
||||
const out = { window: events.length ? { from: new Date(events[0].t).toISOString(), to: new Date(events.at(-1).t).toISOString() } : null };
|
||||
const reqs = events.filter(e => e.k === 'req');
|
||||
const imgs = reqs.filter(isPageImage), served = imgs.filter(e => e.status === 200 && !e.aborted);
|
||||
out.requests = { total: reqs.length, pageImages: imgs.length, served: served.length,
|
||||
notModified: imgs.filter(e => e.status === 304).length, aborted: imgs.filter(e => e.aborted).length,
|
||||
errors: imgs.filter(e => e.status >= 400).length, keepAliveReuse: imgs.length ? round(imgs.filter(e => e.conn > 1).length / imgs.length) : null };
|
||||
out.serverImage = { total: stats(served.map(e => e.ms)), stat: stats(served.map(e => e.stat)),
|
||||
firstRead: stats(served.map(e => serverPhases(e).firstRead)), disk: stats(served.map(e => e.disk)),
|
||||
netWait: stats(served.map(e => e.netWait)), kbytes: stats(served.map(e => e.bytes / 1024)) };
|
||||
const slowServed = served.filter(e => e.ms >= slow);
|
||||
out.slowServer = { count: slowServed.length, share: served.length ? round(slowServed.length / served.length) : null, by: {} };
|
||||
for (const e of slowServed) { const d = dominant(serverPhases(e)); out.slowServer.by[d] = (out.slowServer.by[d] || 0) + 1; }
|
||||
out.slowServer.worst = slowServed.sort((a, b) => b.ms - a.ms).slice(0, 10)
|
||||
.map(e => ({ at: new Date(e.t).toISOString(), ms: e.ms, ...serverPhases(e), path: e.path }));
|
||||
|
||||
const api = {};
|
||||
for (const e of reqs.filter(e => e.path.startsWith('/api/'))) (api[e.path] ||= []).push(e.ms);
|
||||
out.api = Object.fromEntries(Object.entries(api).map(([p, v]) => [p, stats(v)]).sort((a, b) => (b[1].p99 || 0) - (a[1].p99 || 0)).slice(0, 12));
|
||||
|
||||
// Browser view, matched to the server's record of the same file where possible.
|
||||
const res = events.filter(e => e.k === 'client' && e.e === 'res'), byPath = new Map();
|
||||
const decoded = p => { try { return decodeURIComponent(p); } catch { return p; } };
|
||||
for (const e of imgs) { const key = decoded(e.path); (byPath.get(key) || byPath.set(key, []).get(key)).push(e); }
|
||||
const matched = res.map(r => {
|
||||
const cands = (byPath.get(r.url) || []).filter(s => s.t <= r.t + 1000 && s.t >= r.t - 120000);
|
||||
const s = cands.at(-1);
|
||||
return { ...r, serverMs: s ? s.ms : null };
|
||||
});
|
||||
const loaded = matched.filter(r => !r.cached);
|
||||
out.browserImage = { total: stats(loaded.map(r => r.total)), queue: stats(loaded.map(r => r.queue)),
|
||||
ttfb: stats(loaded.map(r => r.ttfb)), download: stats(loaded.map(r => r.download)),
|
||||
connect: stats(loaded.map(r => r.connect)), dns: stats(loaded.map(r => r.dns)), fromCache: matched.length - loaded.length };
|
||||
const slowBrowser = loaded.filter(r => r.total >= slow);
|
||||
out.slowBrowser = { count: slowBrowser.length, share: loaded.length ? round(slowBrowser.length / loaded.length) : null, by: {} };
|
||||
for (const r of slowBrowser) {
|
||||
const parts = { server: r.serverMs ?? 0, network: Math.max(0, (r.ttfb ?? 0) + (r.download ?? 0) - (r.serverMs ?? 0)),
|
||||
queue: r.queue ?? 0, connect: (r.connect ?? 0) + (r.dns ?? 0) };
|
||||
const d = r.serverMs == null ? 'unmatched' : dominant(parts);
|
||||
out.slowBrowser.by[d] = (out.slowBrowser.by[d] || 0) + 1;
|
||||
}
|
||||
const views = events.filter(e => e.k === 'client' && e.e === 'view'), stalls = events.filter(e => e.k === 'client' && e.e === 'stall');
|
||||
out.pageTurns = { count: views.length, preloaded: views.filter(v => v.preloaded).length, wait: stats(views.map(v => v.wait)),
|
||||
over300ms: views.filter(v => v.wait >= 300).length, over1s: views.filter(v => v.wait >= 1000).length };
|
||||
out.flowStalls = { count: stalls.length, wait: stats(stalls.map(s => s.wait)), over1s: stalls.filter(s => s.wait >= 1000).length };
|
||||
out.clients = [...new Set(events.filter(e => e.k === 'client' && e.e === 'hello').map(e => `${e.user}: ${e.ua}`))].slice(0, 10);
|
||||
|
||||
// Storage probe: cold reads of random pages; a long read after quiet time points at disks waking up.
|
||||
const nas = events.filter(e => e.k === 'nas' && !e.error);
|
||||
// Library reads that finished before a probe started (a probe's own read does not count).
|
||||
const ioEnds = events.filter(e => isPageImage(e) || e.k === 'nas' || e.k === 'ahead').map(e => e.t);
|
||||
out.nas = { probes: nas.length, errors: events.filter(e => e.k === 'nas' && e.error).length, first: stats(nas.map(e => e.firstMs)), total: stats(nas.map(e => e.ms)) };
|
||||
out.nas.slow = nas.filter(e => e.firstMs >= 1000).map(e => {
|
||||
const start = e.t - e.ms, before = ioEnds.filter(t => t < start).at(-1);
|
||||
return { at: new Date(e.t).toISOString(), firstMs: e.firstMs, quietMinBefore: before ? round((start - before) / 60000) : null };
|
||||
}).slice(0, 30);
|
||||
|
||||
const loops = events.filter(e => e.k === 'loop');
|
||||
out.eventLoop = { samples: loops.length, lagP99: stats(loops.map(e => e.lagP99)), lagMax: stats(loops.map(e => e.lagMax)), cpu: stats(loops.map(e => e.cpu)), rssMB: stats(loops.map(e => e.rss)) };
|
||||
|
||||
const scans = events.filter(e => e.k === 'scan');
|
||||
out.scans = { count: scans.length, ms: stats(scans.map(e => e.ms)) };
|
||||
const during = e => scans.some(s => e.t >= s.t - s.ms && e.t - e.ms <= s.t);
|
||||
out.scans.slowImagesDuringScan = slowServed.filter(during).length;
|
||||
const span = events.length ? events.at(-1).t - events[0].t : 0;
|
||||
out.scans.timeShare = span ? round(scans.reduce((a, s) => a + s.ms, 0) / span) : null;
|
||||
|
||||
const ahead = events.filter(e => e.k === 'ahead');
|
||||
out.readAhead = { batches: ahead.length, replaced: ahead.filter(e => e.replaced).length, files: ahead.reduce((a, e) => a + e.done, 0),
|
||||
MBps: stats(ahead.filter(e => e.ms > 0 && e.bytes > 0).map(e => round(e.bytes / 1048.576 / e.ms))) };
|
||||
const scraper = events.filter(e => e.k === 'scraper');
|
||||
out.scraper = { calls: scraper.length, errors: scraper.filter(e => e.error).length, ms: stats(scraper.map(e => e.ms)) };
|
||||
out.verdict = verdict(out, slow);
|
||||
return out;
|
||||
}
|
||||
|
||||
function verdict(o, slow) {
|
||||
const lines = [];
|
||||
const top = by => Object.entries(by).sort((a, b) => b[1] - a[1]);
|
||||
if (o.slowServer.count) {
|
||||
const [[phase, n]] = top(o.slowServer.by);
|
||||
const name = { stat: 'the file lookup (stat) on the storage', open: 'opening the file', firstRead: 'the first read from storage',
|
||||
readRest: 'reading the rest of the file from storage', netWait: 'the client receiving data (network)', tail: 'flushing to the client' }[phase];
|
||||
lines.push(`${o.slowServer.count} of ${o.serverImage.total.n} served pages took >= ${slow} ms on the server; ${n} of them mostly in ${name}.`);
|
||||
} else if (o.serverImage.total.n) lines.push(`No served page took >= ${slow} ms on the server (p99 ${o.serverImage.total.p99} ms).`);
|
||||
if (o.slowBrowser.count) {
|
||||
const [[part, n]] = top(o.slowBrowser.by);
|
||||
lines.push(`${o.slowBrowser.count} of ${o.browserImage.total.n} page loads took >= ${slow} ms in the browser; mostly ${part} (${n}).`);
|
||||
}
|
||||
if (o.pageTurns.count) lines.push(`Page turns: ${o.pageTurns.preloaded}/${o.pageTurns.count} preloaded, ${o.pageTurns.over300ms} waited >= 300 ms, ${o.pageTurns.over1s} >= 1 s (p90 ${o.pageTurns.wait.p90 ?? 0} ms).`);
|
||||
if (o.flowStalls.count) lines.push(`Flow mode: ${o.flowStalls.count} pages were on screen before their image (p90 ${o.flowStalls.wait.p90} ms, ${o.flowStalls.over1s} >= 1 s).`);
|
||||
if (o.nas.probes) {
|
||||
lines.push(`Storage probe (cold page): first byte p50 ${o.nas.first.p50} ms, p99 ${o.nas.first.p99} ms, max ${o.nas.first.max} ms; ${o.nas.slow.length} probes >= 1 s.`);
|
||||
const sleepy = o.nas.slow.filter(s => s.quietMinBefore >= 5).length;
|
||||
if (sleepy) lines.push(`${sleepy} slow probes followed >= 5 min without library reads: disks waking from standby is likely.`);
|
||||
}
|
||||
if (o.eventLoop.samples && (o.eventLoop.lagMax.max || 0) >= 200) lines.push(`Event loop blocked up to ${o.eventLoop.lagMax.max} ms (reader CPU work holds requests back).`);
|
||||
if (o.scans.slowImagesDuringScan) lines.push(`${o.scans.slowImagesDuringScan} slow pages overlapped a library scan (scans ran ${Math.round((o.scans.timeShare || 0) * 100)}% of the time).`);
|
||||
return lines;
|
||||
}
|
||||
|
||||
function print(o) {
|
||||
const s = x => (x && x.n ? `n=${x.n} p50=${x.p50} p90=${x.p90} p99=${x.p99} max=${x.max}` : 'n=0');
|
||||
const out = [];
|
||||
out.push(`Trace ${o.window ? `${o.window.from} .. ${o.window.to}` : '(empty)'}`, '', 'Verdict:', ...o.verdict.map(l => ' - ' + l), '');
|
||||
out.push('Requests', ` ${JSON.stringify(o.requests)}`);
|
||||
out.push('Server, page images (ms)');
|
||||
for (const k of ['total', 'stat', 'firstRead', 'disk', 'netWait', 'kbytes']) out.push(` ${k.padEnd(10)} ${s(o.serverImage[k])}`);
|
||||
out.push(` slow by phase ${JSON.stringify(o.slowServer.by)}`);
|
||||
for (const w of o.slowServer.worst) out.push(` ${w.at} ${w.ms} ms stat=${w.stat} first=${w.firstRead} rest=${w.readRest} net=${w.netWait} ${w.path}`);
|
||||
out.push('Browser, page images (ms)');
|
||||
for (const k of ['total', 'queue', 'ttfb', 'download', 'connect', 'dns']) out.push(` ${k.padEnd(10)} ${s(o.browserImage[k])}`);
|
||||
out.push(` from browser cache ${o.browserImage.fromCache}; slow by part ${JSON.stringify(o.slowBrowser.by)}`);
|
||||
out.push(`Page turns ${o.pageTurns.count} (${o.pageTurns.preloaded} preloaded) wait ${s(o.pageTurns.wait)}`);
|
||||
out.push(`Flow stalls ${o.flowStalls.count} wait ${s(o.flowStalls.wait)}`);
|
||||
out.push(`Storage probe first byte ${s(o.nas.first)} total ${s(o.nas.total)} errors ${o.nas.errors}`);
|
||||
for (const p of o.nas.slow) out.push(` ${p.at} first=${p.firstMs} ms after ${p.quietMinBefore ?? '?'} min quiet`);
|
||||
out.push(`Event loop lagP99 ${s(o.eventLoop.lagP99)} lagMax ${s(o.eventLoop.lagMax)} cpu% ${s(o.eventLoop.cpu)} rssMB ${s(o.eventLoop.rssMB)}`);
|
||||
out.push(`Scans ${o.scans.count} ${s(o.scans.ms)}`);
|
||||
out.push(`Read-ahead ${JSON.stringify(o.readAhead)}`);
|
||||
out.push(`Scraper calls ${o.scraper.calls} errors ${o.scraper.errors} ${s(o.scraper.ms)}`);
|
||||
out.push('API (ms, slowest p99 first)');
|
||||
for (const [p, v] of Object.entries(o.api)) out.push(` ${p.padEnd(28)} ${s(v)}`);
|
||||
if (o.clients.length) out.push('Clients', ...o.clients.map(c => ' ' + c));
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
const round = v => Math.round(v * 100) / 100;
|
||||
|
||||
if (require.main === module) {
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
if (opts.help || !opts.files.length) { console.log('usage: node tools/perf-report.js <trace.jsonl|folder>... [--since ISO] [--until ISO] [--slow ms] [--json]'); process.exit(opts.help ? 0 : 2); }
|
||||
const result = analyse(readEvents(opts.files, opts), opts);
|
||||
console.log(opts.json ? JSON.stringify(result, null, 2) : print(result));
|
||||
}
|
||||
|
||||
module.exports = { analyse, readEvents, print };
|
||||
Reference in new issue
Block a user