From 57a571d2f074a20dcd34f7369a72129b520e069c Mon Sep 17 00:00:00 2001 From: maashkinoozhe Date: Sun, 25 Oct 2026 07:47:47 +0100 Subject: [PATCH] WeebCentral and MangaPill are off by default, with an admin switch per source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are unofficial sites that mostly host unlicensed copies; MangaDex (official API) stays on. Settings → Admin → Scraper → Hoster settings has a 'Use ' switch. A switched-off source refuses new jobs, keeps queued jobs waiting and is skipped by nightly update checks. Installations whose queue or history already used a source keep it switched on, so upgrades don't stop a working queue. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 ++++-- docs/deployment.md | 6 +++++- docs/scraper-service.md | 2 +- public/settings.js | 14 +++++++++++-- scraper/service.js | 39 ++++++++++++++++++++++++++++++++---- scraper/test/scraper.test.js | 27 +++++++++++++++++++++++++ 6 files changed, 84 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 898c78d..37afbd3 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,10 @@ URLs, so a sub-path like `/manga/` works. ### Sources and copyright -The scraper can download from WeebCentral, MangaDex and MangaPill, and peering can copy chapters between -servers. You are responsible for only downloading and sharing what you are allowed to copy where you live, +The scraper can download from MangaDex (on by default; official API) and from WeebCentral and MangaPill. +The last two are unofficial sites that mostly host unlicensed copies, so they are **off by default**. An admin +can switch each one on under **Settings → Admin → Scraper → Hoster settings**. Peering can copy chapters +between servers. You are responsible for only downloading and sharing what you are allowed to copy where you live, and for respecting each site's terms. The scraper is deliberately slow and stops on access errors; it never bypasses CAPTCHAs, logins or other access controls ([scraper/AGENT_RUNBOOK.md](scraper/AGENT_RUNBOOK.md)). diff --git a/docs/deployment.md b/docs/deployment.md index c051986..2add36a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -114,7 +114,11 @@ account. The library page fills as soon as the first scan finishes. - **Private access over a VPN** (WireGuard, Tailscale, …): bind the reader to the VPN address, or keep it on loopback and forward from the VPN client. - **Adding manga:** admins use **Settings → Admin → Scraper → Inspect & queue** with a series URL from a - supported source (WeebCentral, MangaDex, MangaPill), or copy folders into the library by hand. The reader + supported source, or copy folders into the library by hand. MangaDex is on by default. WeebCentral and + MangaPill are unofficial sites that mostly host unlicensed copies, so they start switched off. Switch them + on per source under **Hoster settings** if you may copy what you download there. While a source is off, + its queued jobs wait and nightly update checks skip it. Upgraded installations keep the sources their + queue already used. The reader notices new folders within `SCAN_INTERVAL` minutes. Rules for tools that write into the library: [scraper-guide.md](scraper-guide.md). Only download what you are allowed to copy; the scraper keeps conservative per-host rate limits and stops on 401/403/429 ([scraper/AGENT_RUNBOOK.md](../scraper/AGENT_RUNBOOK.md)). diff --git a/docs/scraper-service.md b/docs/scraper-service.md index 3a3828f..51d8506 100644 --- a/docs/scraper-service.md +++ b/docs/scraper-service.md @@ -205,7 +205,7 @@ All responses are JSON. When `SCRAPER_TOKEN` is set, send | `POST` | `/api/queue/clear-completed` | `{source?}`; remove inactive successful download job rows only; preserve library, history, checkpoints and other statuses | | `POST` | `/api/queue/reorder` | `{ids}` containing every queue id exactly once | | `POST` | `/api/control` | `{action:"pause"|"resume"|"retry"|"inspect"|"confirm"|"skip",id?}`; `pause-source`/`resume-source` use `id` as the source name and do not change the global pause | -| `POST` | `/api/settings` | `sourceRates.{adapter}.{minMs,maxMs,chapterMinMs,chapterMaxMs}` plus `autoUpdateEnabled` and local `autoUpdateTime` (`HH:MM`) | +| `POST` | `/api/settings` | `sourceRates.{adapter}.{minMs,maxMs,chapterMinMs,chapterMaxMs}`, `sourcesEnabled.{weebcentral,mangadex,mangapill}` (booleans; WeebCentral and MangaPill default to off, except on installations whose queue already used them; a switched-off source refuses new jobs, keeps queued ones waiting and is skipped by update checks), plus `autoUpdateEnabled` and local `autoUpdateTime` (`HH:MM`) | | `POST` | `/api/discovery/run` | Queue an immediate update check for every enabled manga | | `POST` | `/api/library/auto-update` | `{id,enabled}` updates `manga.json` atomically | | `POST` | `/api/legacy/publish-ready` | Paused-only validation/publication of completed legacy work; converts its checkpoint to incremental mode | diff --git a/public/settings.js b/public/settings.js index ae8e7e9..2deb48d 100644 --- a/public/settings.js +++ b/public/settings.js @@ -206,7 +206,8 @@ return; } const activeEntries = state.active?.length ? state.active : state.current ? [{ source: state.current.source, current: state.current, job: state.job }] : []; - const url = h('input', { type: 'url', placeholder: 'WeebCentral or MangaDex title URL', class: 'grow' }); + const usable = (state.adapters || []).filter(adapter => state.settings?.sourcesEnabled?.[adapter.id] !== false).map(adapter => adapter.label); + const url = h('input', { type: 'url', placeholder: `${usable.join(', ') || 'Source'} title URL`, class: 'grow' }); const title = h('input', { placeholder: 'Title override (optional)' }); const scope = h('select', {}, h('option', { value: 'missing' }, 'All missing chapters'), @@ -229,8 +230,16 @@ const max = h('input', { type: 'number', min: '.5', step: '.5', value: (sourceRate.maxMs ?? 4000) / 1000, title: 'Maximum seconds between image and API requests' }); const chapterMin = h('input', { type: 'number', min: '.5', step: '1', value: (sourceRate.chapterMinMs ?? 5000) / 1000, title: 'Minimum seconds between chapters' }); const chapterMax = h('input', { type: 'number', min: '.5', step: '1', value: (sourceRate.chapterMaxMs ?? 15000) / 1000, title: 'Maximum seconds between chapters' }); + // Sources with a switch (sourcesEnabled) start switched off on new installations. + const switchable = Object.hasOwn(rate.sourcesEnabled || {}, adapter.id); + const enabled = switchable ? h('input', { type: 'checkbox', checked: rate.sourcesEnabled[adapter.id] === true, + onchange: e => command({ command: 'settings', settings: { sourcesEnabled: { [adapter.id]: e.target.checked } } }) }) : null; return h('div', { class: 'source-rate' }, h('b', {}, adapter.label), + switchable ? h('label', { class: 'check' }, enabled, h('span', {}, `Use ${adapter.label}`, + h('div', { class: 'muted small' }, adapter.id === 'mangadex' + ? 'Official API; titles removed at a publisher\'s request are not offered.' + : 'Unofficial site that mostly hosts unlicensed copies. Switch it on only for what you may copy. While it is off, its queued jobs wait and update checks skip it.'))) : null, h('div', { class: 'row rate-row' }, h('label', {}, h('small', {}, 'Requests min (s)'), min), h('label', {}, h('small', {}, 'Requests max (s)'), max), h('label', {}, h('small', {}, 'Chapters min (s)'), chapterMin), h('label', {}, h('small', {}, 'Chapters max (s)'), chapterMax), @@ -248,7 +257,8 @@ const completed = visible(state.queue.filter(item => item.status === 'complete' && item.kind === 'download')); const when = value => value && !Number.isNaN(Date.parse(value)) ? new Date(value).toLocaleString() : 'Never'; const chips = h('div', { class: 'row scraper-filters', role: 'group', 'aria-label': 'Filter by source' }, - [{ id: '', label: 'All sources' }, ...(state.adapters || []).map(adapter => ({ id: adapter.id, label: adapter.label }))].map(option => + [{ id: '', label: 'All sources' }, ...(state.adapters || []).map(adapter => ({ id: adapter.id, + label: adapter.label + (state.settings?.sourcesEnabled?.[adapter.id] === false ? ' (off)' : '') }))].map(option => h('button', { class: 'btn small', type: 'button', 'aria-pressed': sourceFilter === option.id ? 'true' : 'false', onclick: () => { sourceFilter = option.id; draw(lastState); } }, option.label))); card.append(chips); diff --git a/scraper/service.js b/scraper/service.js index f592f1b..6811960 100644 --- a/scraper/service.js +++ b/scraper/service.js @@ -24,9 +24,13 @@ const DEFAULT_SOURCE_RATES = { mangadex: { ...FAST_RATE_SETTINGS }, mangapill: { ...FAST_RATE_SETTINGS }, }; +// Sources an admin must switch on first. WeebCentral and MangaPill mostly host unlicensed +// copies; MangaDex has an official API and handles takedowns. Peers are always available. +const DEFAULT_SOURCES_ENABLED = { weebcentral: false, mangadex: true, mangapill: false }; const DEFAULT_SETTINGS = { ...FAST_RATE_SETTINGS, sourceRates: Object.fromEntries(Object.entries(DEFAULT_SOURCE_RATES).map(([source, rate]) => [source, { ...rate }])), + sourcesEnabled: { ...DEFAULT_SOURCES_ENABLED }, autoUpdateEnabled: true, autoUpdateTime: '03:00', }; @@ -73,6 +77,15 @@ function validateSettings(input = {}, prior = DEFAULT_SETTINGS) { }; next.sourceRates[source] = validateRateSettings(patch || {}, base, `sourceRates.${source}.`); } + if (input.sourcesEnabled !== undefined && (!input.sourcesEnabled || typeof input.sourcesEnabled !== 'object' || Array.isArray(input.sourcesEnabled))) { + throw new Error('sourcesEnabled must be an object'); + } + next.sourcesEnabled = { ...DEFAULT_SOURCES_ENABLED, ...(prior.sourcesEnabled || {}) }; + for (const [source, enabled] of Object.entries(input.sourcesEnabled || {})) { + if (!Object.hasOwn(DEFAULT_SOURCES_ENABLED, source)) throw new Error(`Unknown source: ${source}`); + if (typeof enabled !== 'boolean') throw new Error(`sourcesEnabled.${source} must be true or false`); + next.sourcesEnabled[source] = enabled; + } if (input.autoUpdateEnabled !== undefined) next.autoUpdateEnabled = !!input.autoUpdateEnabled; if (input.autoUpdateTime !== undefined) { const value = String(input.autoUpdateTime).trim(); @@ -86,6 +99,13 @@ function settingsForSource(settings, source) { return { ...settings, ...((settings.sourceRates || {})[source] || {}) }; } +// Installations from before the switch keep every source they already use: off-by-default +// applies to new installations, it must not silently stop a working queue. +function inheritedSourcesEnabled(saved) { + const used = new Set([...(saved.queue || []), ...(saved.history || [])].map(item => item && item.source)); + return Object.fromEntries(Object.entries(DEFAULT_SOURCES_ENABLED).map(([source, enabled]) => [source, enabled || used.has(source)])); +} + function existingPolicy(value) { const policy = String(value || 'skip-any'); if (!['skip-any', 'add-source'].includes(policy)) throw new Error('existingPolicy must be skip-any or add-source'); @@ -217,11 +237,13 @@ class ScraperService { async loadState() { if (await exists(this.stateFile)) { const saved = JSON.parse(await fs.readFile(this.stateFile, 'utf8')); + const savedSettings = saved.settings || {}; + if (!savedSettings.sourcesEnabled) savedSettings.sourcesEnabled = inheritedSourcesEnabled(saved); return { version: 3, paused: false, status: 'idle', queue: [], history: [], publishedRevision: 0, currentIds: {}, sourcePaused: {}, settings: DEFAULT_SETTINGS, ...saved, - settings: validateSettings(saved.settings || {}, DEFAULT_SETTINGS), + settings: validateSettings(savedSettings, DEFAULT_SETTINGS), }; } const state = { @@ -239,7 +261,7 @@ class ScraperService { status: completed.has(item.id) ? 'complete' : 'queued', order: index, added: legacy.created || new Date().toISOString(), })); - state.settings = validateSettings(legacy.settings || {}, DEFAULT_SETTINGS); + state.settings = validateSettings({ ...(legacy.settings || {}), sourcesEnabled: inheritedSourcesEnabled(state) }, DEFAULT_SETTINGS); state.migratedFrom = this.legacyStateFile; } return state; @@ -401,7 +423,8 @@ class ScraperService { let queued = 0, skipped = 0; for (const manga of inventory.mangas) { if (!manga.autoUpdate) continue; - const supported = manga.sources.map(source => ({ source, found: this.resolveAdapter(source.url) })).filter(item => item.found); + const supported = manga.sources.map(source => ({ source, found: this.resolveAdapter(source.url) })) + .filter(item => item.found && this.sourceEnabled(item.found.adapter.id)); if (!supported.length) { skipped++; continue; } const preferred = manga.meta?.['x-scraper']?.preferredSource; const selected = supported.find(item => item.found.adapter.id === preferred) || supported[0]; @@ -598,9 +621,16 @@ class ScraperService { }; } + sourceEnabled(source) { + return !Object.hasOwn(DEFAULT_SOURCES_ENABLED, source) || this.state.settings.sourcesEnabled?.[source] === true; + } + async enqueue(input) { const found = this.resolveAdapter(input.url); if (!found) throw new Error('No installed scraper adapter accepts this URL'); + if (!this.sourceEnabled(found.adapter.id)) { + throw new Error(`${found.adapter.label || found.adapter.id} is switched off. An admin can switch it on under Settings → Admin → Scraper → Hoster settings.`); + } const workIdentity = normalizeWorkIdentity(input.workIdentity); if (input.workIdentity != null && !workIdentity) throw new Error('Invalid manga work identity'); const scope = normalizeScope(input.scope || {}); @@ -902,7 +932,8 @@ class ScraperService { continue; } const source = found.adapter.id; - if (this.active.has(source) || this.state.sourcePaused[source]) continue; + // A switched-off source keeps its queued jobs; they wait until it is switched on again. + if (this.active.has(source) || this.state.sourcePaused[source] || !this.sourceEnabled(source)) continue; if (this.state.circuits[source]?.state === 'open') continue; if (item.libraryMangaId && [...this.active.values()].some(entry => { const activeItem = this.state.queue.find(candidate => candidate.id === entry.id); diff --git a/scraper/test/scraper.test.js b/scraper/test/scraper.test.js index efcb392..c46d2af 100644 --- a/scraper/test/scraper.test.js +++ b/scraper/test/scraper.test.js @@ -426,6 +426,9 @@ test('daemon queue state is durable while paused', async t => { })); const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init(); t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); }); + // WeebCentral is off on a fresh installation until an admin switches it on. + await assert.rejects(() => service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' }), /switched off/); + await service.updateSettings({ sourcesEnabled: { weebcentral: true } }); const item = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' }); assert.equal(item.status, 'queued'); const scoped = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series', scope: { mode: 'latest' } }); @@ -439,6 +442,30 @@ test('daemon queue state is durable while paused', async t => { assert.equal((await service.publicState()).queue.length, 0); }); +test('sources are off by default but existing installations keep the ones they use', async t => { + assert.deepEqual(validateSettings({}).sourcesEnabled, { weebcentral: false, mangadex: true, mangapill: false }); + assert.throws(() => validateSettings({ sourcesEnabled: { weebcentral: 'yes' } }), /true or false/); + assert.throws(() => validateSettings({ sourcesEnabled: { other: true } }), /Unknown source/); + const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-sources-')); + const dataRoot = path.join(temp, 'data'); + await fs.mkdir(dataRoot, { recursive: true }); + const { sourcesEnabled, ...older } = validateSettings({}); + await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ + version: 3, paused: true, status: 'paused', settings: older, publishedRevision: 0, + queue: [{ id: 'A', title: 'A', url: 'https://weebcentral.com/series/AAA/A', source: 'weebcentral', status: 'queued', order: 0 }], + history: [], + })); + const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init(); + t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); }); + assert.deepEqual(service.state.settings.sourcesEnabled, { weebcentral: true, mangadex: true, mangapill: false }); + await assert.rejects(() => service.enqueue({ url: 'https://mangapill.com/manga/2/one-piece' }), /MangaPill is switched off/); + // Switching a source off keeps its jobs queued instead of failing them. + await service.updateSettings({ sourcesEnabled: { weebcentral: false } }); + assert.equal(service.sourceEnabled('weebcentral'), false); + assert.equal(service.sourceEnabled('peer-abc'), true, 'peers have no switch'); + assert.equal(service.state.queue[0].status, 'queued'); +}); + test('retry recovers stale downloading entries before work resumes', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-retry-')); const dataRoot = path.join(temp, 'data');