diff --git a/discovery/anilist.js b/discovery/anilist.js new file mode 100644 index 0000000..184dcc2 --- /dev/null +++ b/discovery/anilist.js @@ -0,0 +1,146 @@ +'use strict'; + +// Polite AniList GraphQL client for discovery seeds. Sequential, rate limited, stops on access controls. + +const STAFF_ROLE = /^(?:original story|story(?: & art)?|art|illustration)$/i; +const USER_AGENT = 'MangaCircus-Discovery/1.0'; + +// Stop: the whole run should end. Skip: this one request is lost, carry on. +class AniListStop extends Error { + constructor(reason, message) { super(message || `AniList stop: ${reason}`); this.name = 'AniListStop'; this.reason = reason; } +} +class AniListSkip extends Error { + constructor(reason, message) { super(message || `AniList skip: ${reason}`); this.name = 'AniListSkip'; this.reason = reason; } +} + +function createLimiter({ perMinute = 25, now = Date.now, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) { + const interval = 60000 / perMinute; + let nextAllowed = 0; + let tail = Promise.resolve(); + // Chained so concurrent callers queue up instead of all reading the same clock. + function wait() { + const turn = tail.then(async () => { + await sleep(Math.max(0, nextAllowed - now())); + nextAllowed = now() + interval; + }); + tail = turn.catch(() => {}); + return turn; + } + return { wait }; +} + +function normalizeMedia(media) { + if (!media || (media.format !== 'MANGA' && media.format !== 'ONE_SHOT')) return null; + const title = media.title || {}; + const names = [title.english || title.romaji || title.native, title.romaji, title.native, ...(media.synonyms || [])].filter(Boolean); + return { + anilistId: media.id, + title: names[0] || '', + altTitles: [...new Set(names.slice(1))].filter(name => name !== names[0]), + format: media.format, + isAdult: !!media.isAdult, + status: media.status === 'RELEASING' ? 'ongoing' : media.status === 'FINISHED' ? 'completed' : 'unknown', + year: media.startDate?.year ?? null, + averageScore: media.averageScore ?? null, + popularity: media.popularity ?? null, + trending: media.trending ?? null, + tags: [ + ...(media.genres || []).map(name => ({ name, rank: 100, kind: 'genre', isAdult: false })), + ...(media.tags || []).map(tag => ({ name: tag.name, rank: tag.rank, kind: 'tag', isAdult: !!tag.isAdult }))], + authors: (media.staff?.edges || []) + .filter(edge => STAFF_ROLE.test(edge.role || '') && edge.node?.name?.full) + .map(edge => ({ staffId: edge.node.id, name: edge.node.name.full, role: edge.role })), + coverUrl: media.coverImage?.extraLarge || null, + siteUrl: media.siteUrl || null, + }; +} + +const MEDIA_FIELDS = `fragment MediaFields on Media { + id format isAdult title { romaji english native } synonyms status startDate { year } + averageScore popularity trending genres tags { name rank isAdult } + staff(perPage: 20) { edges { role node { id name { full } } } } + coverImage { extraLarge } siteUrl +}`; + +const QUERIES = { + title: `query ($id: Int) { Media(id: $id, type: MANGA) { ...MediaFields } } ${MEDIA_FIELDS}`, + recs: `query ($id: Int) { Media(id: $id, type: MANGA) { + recommendations(perPage: 25, sort: RATING_DESC) { nodes { rating mediaRecommendation { ...MediaFields } } } } } ${MEDIA_FIELDS}`, + creator: `query ($id: Int) { Staff(id: $id) { + staffMedia(type: MANGA, perPage: 25, sort: POPULARITY_DESC) { edges { staffRole node { ...MediaFields } } } } } ${MEDIA_FIELDS}`, + tagTopGenre: `query ($genre_in: [String]) { Page(perPage: 25) { + media(type: MANGA, genre_in: $genre_in, popularity_greater: 1000, sort: POPULARITY_DESC) { ...MediaFields } } } ${MEDIA_FIELDS}`, + tagTopTag: `query ($tag_in: [String]) { Page(perPage: 25) { + media(type: MANGA, tag_in: $tag_in, popularity_greater: 1000, sort: POPULARITY_DESC) { ...MediaFields } } } ${MEDIA_FIELDS}`, + fresh: `query ($startDate_greater: FuzzyDateInt, $perPage: Int) { Page(perPage: $perPage) { + media(type: MANGA, status: RELEASING, startDate_greater: $startDate_greater, sort: TRENDING_DESC) { ...MediaFields } } } ${MEDIA_FIELDS}`, +}; + +const isTimeout = error => error && (error.name === 'TimeoutError' || error.name === 'AbortError'); +const hasId = media => media && Number.isInteger(media.id); +const need = (value, check = Array.isArray) => { if (!check(value)) throw new AniListSkip('invalid', 'AniList response is missing data'); return value; }; + +function createClient({ endpoint = process.env.DISCOVERY_ANILIST_URL || 'https://graphql.anilist.co', limiter = createLimiter(), + fetchImpl = fetch, timeoutMs = 30000, today = () => new Date() } = {}) { + async function post(query, variables) { + await limiter.wait(); + let response, text; + try { + response = await fetchImpl(endpoint, { + method: 'POST', headers: { 'content-type': 'application/json', 'user-agent': USER_AGENT }, + body: JSON.stringify({ query, variables }), signal: AbortSignal.timeout(timeoutMs) }); + if (response.status === 429) throw new AniListStop('rate-limited'); + if (response.status === 401 || response.status === 403) throw new AniListStop('forbidden'); + if (response.status >= 500) throw new AniListSkip('upstream', `AniList HTTP ${response.status}`); + text = await response.text(); + } catch (error) { + if (error instanceof AniListStop || error instanceof AniListSkip) throw error; + if (isTimeout(error)) throw new AniListSkip('timeout'); + throw new AniListSkip('upstream', `AniList request failed: ${error.message}`); + } + // An HTML or non-JSON page means a bot-check in front of the API; never work around it. + if (/text\/html/i.test(response.headers.get('content-type') || '')) throw new AniListStop('challenge'); + let body; + try { body = JSON.parse(text); } catch { throw new AniListStop('challenge'); } + if (!response.ok || !body || typeof body !== 'object') throw new AniListSkip('invalid', `AniList HTTP ${response.status}`); + return body.data || {}; + } + + const records = (list, pick) => need(list).filter(hasId).map(item => ({ item, record: normalizeMedia(pick(item)) })).filter(entry => entry.record); + const mediaOf = data => { need(data.Media, value => value && typeof value === 'object'); return data.Media; }; + + return { + async title(id) { + const media = mediaOf(await post(QUERIES.title, { id })); + if (!hasId(media)) throw new AniListSkip('invalid'); + return normalizeMedia(media); + }, + async recs(id) { + const nodes = need(mediaOf(await post(QUERIES.recs, { id })).recommendations?.nodes); + return nodes.filter(node => node.rating > 0 && hasId(node.mediaRecommendation)) + .map(node => ({ anilistId: node.mediaRecommendation.id, rating: node.rating, record: normalizeMedia(node.mediaRecommendation) })) + .filter(entry => entry.record); + }, + async creator(staffId) { + const data = await post(QUERIES.creator, { id: staffId }); + const edges = need(need(data.Staff, value => value && typeof value === 'object').staffMedia?.edges); + return edges.filter(edge => hasId(edge.node)) + .map(edge => ({ anilistId: edge.node.id, role: edge.staffRole, record: normalizeMedia(edge.node) })) + .filter(entry => entry.record); + }, + async tagTop({ name, kind }) { + const genre = kind === 'genre'; + const data = await post(genre ? QUERIES.tagTopGenre : QUERIES.tagTopTag, genre ? { genre_in: [name] } : { tag_in: [name] }); + return records(data.Page?.media, media => media).map(({ item, record }) => ({ anilistId: item.id, record })); + }, + async fresh() { + const date = today(); + // AniList FuzzyDateInt: YYYYMMDD, two years back. + const startDate = (date.getUTCFullYear() - 2) * 10000 + (date.getUTCMonth() + 1) * 100 + date.getUTCDate(); + const data = await post(QUERIES.fresh, { startDate_greater: startDate, perPage: 50 }); + return records(data.Page?.media, media => media).map(({ item, record }) => ({ anilistId: item.id, record })); + }, + }; +} + +module.exports = { createLimiter, createClient, normalizeMedia, AniListStop, AniListSkip }; diff --git a/test/discovery-anilist.test.js b/test/discovery-anilist.test.js new file mode 100644 index 0000000..fc98aac --- /dev/null +++ b/test/discovery-anilist.test.js @@ -0,0 +1,113 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { createLimiter, createClient, normalizeMedia, AniListStop, AniListSkip } = require('../discovery/anilist'); + +const fixture = name => JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures', 'anilist', `${name}.json`), 'utf8')); +const json = (body, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); +const noWait = { wait: async () => {} }; + +function stub(respond) { + const calls = []; + const fetchImpl = async (url, init) => { calls.push({ url, init, body: JSON.parse(init.body) }); return respond(); }; + return { calls, client: createClient({ endpoint: 'http://localhost/graphql', limiter: noWait, fetchImpl, today: () => new Date('2026-10-25') }) }; +} +const clientFor = respond => stub(respond).client; + +test('normalizeMedia maps fields and drops novels', () => { + const media = fixture('title').data.Media; + assert.deepEqual(normalizeMedia(media), { + anilistId: 101, title: 'Moon Traveller', altTitles: ['Tsuki no Tabibito', '月の旅人', 'Lunar Wanderer'], + format: 'MANGA', isAdult: false, status: 'ongoing', year: 2019, averageScore: 81, popularity: 5400, trending: 37, + tags: [ + { name: 'Adventure', rank: 100, kind: 'genre', isAdult: false }, + { name: 'Fantasy', rank: 100, kind: 'genre', isAdult: false }, + { name: 'Travel', rank: 88, kind: 'tag', isAdult: false }, + { name: 'Nudity', rank: 40, kind: 'tag', isAdult: true }], + authors: [{ staffId: 900, name: 'Aoi Example', role: 'Story & Art' }], + coverUrl: 'https://example.invalid/cover/101.jpg', siteUrl: 'https://anilist.co/manga/101' }); + assert.equal(normalizeMedia({ ...media, format: 'NOVEL' }), null); + assert.equal(normalizeMedia(fixture('novel').data.Media), null); + assert.equal(normalizeMedia({ ...media, status: 'HIATUS' }).status, 'unknown'); +}); + +test('title returns a record and sends the fixed headers', async () => { + const { client, calls } = stub(() => json(fixture('title'))); + const record = await client.title(101); + assert.equal(record.anilistId, 101); + assert.equal(calls[0].body.variables.id, 101); + assert.equal(calls[0].init.headers['user-agent'], 'MangaCircus-Discovery/1.0'); + assert.equal(calls[0].init.headers['content-type'], 'application/json'); +}); + +test('recs keeps positive ratings and manga formats', async () => { + const out = await clientFor(() => json(fixture('recs'))).recs(101); + assert.equal(out.length, 1); + assert.equal(out[0].anilistId, 201); + assert.equal(out[0].rating, 12); + assert.equal(out[0].record.title, 'Sky Town'); +}); + +test('creator returns roles and drops novels', async () => { + const out = await clientFor(() => json(fixture('creator'))).creator(900); + assert.deepEqual(out.map(e => [e.anilistId, e.role]), [[301, 'Story & Art']]); +}); + +test('fresh query sends startDate two years back', async () => { + const { client, calls } = stub(() => json(fixture('fresh'))); + const out = await client.fresh(); + assert.equal(calls[0].body.variables.startDate_greater, 20241025); + assert.equal(calls[0].body.variables.perPage, 50); + assert.match(calls[0].body.query, /TRENDING_DESC/); + assert.deepEqual(out.map(e => e.anilistId), [501]); +}); + +test('tagTop uses genre_in for genres and tag_in for tags', async () => { + const g = stub(() => json(fixture('tagtop'))); + const out = await g.client.tagTop({ name: 'Mystery', kind: 'genre' }); + assert.deepEqual(g.calls[0].body.variables.genre_in, ['Mystery']); + assert.equal(g.calls[0].body.variables.tag_in, undefined); + assert.match(g.calls[0].body.query, /popularity_greater: 1000/); + assert.deepEqual(out.map(e => e.anilistId), [401]); + const t = stub(() => json(fixture('tagtop'))); + await t.client.tagTop({ name: 'Detective', kind: 'tag' }); + assert.deepEqual(t.calls[0].body.variables.tag_in, ['Detective']); + assert.equal(t.calls[0].body.variables.genre_in, undefined); +}); + +test('429 stops, 403 stops, html body is a challenge', async () => { + await assert.rejects(clientFor(() => json({}, 429)).title(1), e => e instanceof AniListStop && e.reason === 'rate-limited'); + await assert.rejects(clientFor(() => json({}, 403)).title(1), e => e instanceof AniListStop && e.reason === 'forbidden'); + await assert.rejects(clientFor(() => json({}, 401)).title(1), e => e instanceof AniListStop && e.reason === 'forbidden'); + const html = () => new Response('checking your browser', { status: 200, headers: { 'content-type': 'text/html' } }); + await assert.rejects(clientFor(html).title(1), e => e instanceof AniListStop && e.reason === 'challenge'); + const garbage = () => new Response('not json', { status: 200 }); + await assert.rejects(clientFor(garbage).title(1), e => e instanceof AniListStop && e.reason === 'challenge'); +}); + +test('5xx and timeout skip', async () => { + await assert.rejects(clientFor(() => json({}, 503)).title(1), e => e instanceof AniListSkip && e.reason === 'upstream'); + const timeout = () => { throw Object.assign(new Error('timed out'), { name: 'TimeoutError' }); }; + await assert.rejects(clientFor(timeout).title(1), e => e instanceof AniListSkip && e.reason === 'timeout'); + const abort = () => { throw Object.assign(new Error('aborted'), { name: 'AbortError' }); }; + await assert.rejects(clientFor(abort).title(1), e => e instanceof AniListSkip && e.reason === 'timeout'); +}); + +test('missing data is invalid', async () => { + const invalid = e => e instanceof AniListSkip && e.reason === 'invalid'; + await assert.rejects(clientFor(() => json({ data: {} })).title(1), invalid); + await assert.rejects(clientFor(() => json({ data: { Page: {} } })).fresh(), invalid); + await assert.rejects(clientFor(() => json({ data: {} })).creator(1), invalid); + await assert.rejects(clientFor(() => json({ data: { Media: { ...fixture('title').data.Media, id: 'x' } } })).title(1), invalid); +}); + +test('limiter spaces calls 2400 ms apart', async () => { + let clock = 1000; + const sleeps = []; + const limiter = createLimiter({ perMinute: 25, now: () => clock, sleep: async ms => { sleeps.push(ms); clock += ms; } }); + await Promise.all([limiter.wait(), limiter.wait(), limiter.wait()]); + assert.deepEqual(sleeps, [0, 2400, 2400]); +}); diff --git a/test/fixtures/anilist/creator.json b/test/fixtures/anilist/creator.json new file mode 100644 index 0000000..36a8e84 --- /dev/null +++ b/test/fixtures/anilist/creator.json @@ -0,0 +1,10 @@ +{"data":{"Staff":{"staffMedia":{"edges":[ + {"staffRole":"Story & Art","node":{"id":301,"format":"MANGA","isAdult":false, + "title":{"romaji":"Hoshi no Uta","english":"Star Song","native":null},"synonyms":[], + "status":"FINISHED","startDate":{"year":2018},"averageScore":79,"popularity":2500,"trending":3, + "genres":["Fantasy"],"tags":[],"staff":{"edges":[]},"coverImage":{"extraLarge":null},"siteUrl":null}}, + {"staffRole":"Original Story","node":{"id":777,"format":"NOVEL","isAdult":false, + "title":{"romaji":"Hon no Tabi","english":null,"native":null},"synonyms":[], + "status":"FINISHED","startDate":{"year":2010},"averageScore":70,"popularity":1200,"trending":1, + "genres":[],"tags":[],"staff":{"edges":[]},"coverImage":{"extraLarge":null},"siteUrl":null}} +]}}}} diff --git a/test/fixtures/anilist/fresh.json b/test/fixtures/anilist/fresh.json new file mode 100644 index 0000000..3531a95 --- /dev/null +++ b/test/fixtures/anilist/fresh.json @@ -0,0 +1,6 @@ +{"data":{"Page":{"media":[ + {"id":501,"format":"MANGA","isAdult":false, + "title":{"romaji":"Atarashii Kaze","english":"New Wind","native":null},"synonyms":[], + "status":"RELEASING","startDate":{"year":2025},"averageScore":72,"popularity":2100,"trending":150, + "genres":["Sports"],"tags":[],"staff":{"edges":[]},"coverImage":{"extraLarge":null},"siteUrl":null} +]}}} diff --git a/test/fixtures/anilist/novel.json b/test/fixtures/anilist/novel.json new file mode 100644 index 0000000..d83a345 --- /dev/null +++ b/test/fixtures/anilist/novel.json @@ -0,0 +1,5 @@ +{"data":{"Media":{"id":777,"format":"NOVEL","isAdult":false, + "title":{"romaji":"Hon no Tabi","english":null,"native":null}, + "synonyms":[],"status":"FINISHED","startDate":{"year":2010}, + "averageScore":70,"popularity":1200,"trending":1,"genres":[],"tags":[], + "staff":{"edges":[]},"coverImage":{"extraLarge":null},"siteUrl":"https://anilist.co/manga/777"}}} diff --git a/test/fixtures/anilist/recs.json b/test/fixtures/anilist/recs.json new file mode 100644 index 0000000..68561d8 --- /dev/null +++ b/test/fixtures/anilist/recs.json @@ -0,0 +1,14 @@ +{"data":{"Media":{"recommendations":{"nodes":[ + {"rating":12,"mediaRecommendation":{"id":201,"format":"MANGA","isAdult":false, + "title":{"romaji":"Sora no Machi","english":"Sky Town","native":null},"synonyms":[], + "status":"FINISHED","startDate":{"year":2015},"averageScore":77,"popularity":3000,"trending":5, + "genres":["Drama"],"tags":[],"staff":{"edges":[]},"coverImage":{"extraLarge":null},"siteUrl":"https://anilist.co/manga/201"}}, + {"rating":-2,"mediaRecommendation":{"id":202,"format":"MANGA","isAdult":false, + "title":{"romaji":"Kuroi Mori","english":null,"native":null},"synonyms":[], + "status":"HIATUS","startDate":{"year":2012},"averageScore":60,"popularity":900,"trending":0, + "genres":[],"tags":[],"staff":{"edges":[]},"coverImage":{"extraLarge":null},"siteUrl":null}}, + {"rating":9,"mediaRecommendation":{"id":777,"format":"NOVEL","isAdult":false, + "title":{"romaji":"Hon no Tabi","english":null,"native":null},"synonyms":[], + "status":"FINISHED","startDate":{"year":2010},"averageScore":70,"popularity":1200,"trending":1, + "genres":[],"tags":[],"staff":{"edges":[]},"coverImage":{"extraLarge":null},"siteUrl":null}} +]}}}} diff --git a/test/fixtures/anilist/tagtop.json b/test/fixtures/anilist/tagtop.json new file mode 100644 index 0000000..22e9769 --- /dev/null +++ b/test/fixtures/anilist/tagtop.json @@ -0,0 +1,11 @@ +{"data":{"Page":{"media":[ + {"id":401,"format":"ONE_SHOT","isAdult":false, + "title":{"romaji":"Ichi Wa","english":"One Tale","native":null},"synonyms":[], + "status":"FINISHED","startDate":{"year":2020},"averageScore":74,"popularity":1800,"trending":2, + "genres":["Mystery"],"tags":[{"name":"Detective","rank":90,"isAdult":false}],"staff":{"edges":[]}, + "coverImage":{"extraLarge":null},"siteUrl":null}, + {"id":777,"format":"NOVEL","isAdult":false, + "title":{"romaji":"Hon no Tabi","english":null,"native":null},"synonyms":[], + "status":"FINISHED","startDate":{"year":2010},"averageScore":70,"popularity":1200,"trending":1, + "genres":[],"tags":[],"staff":{"edges":[]},"coverImage":{"extraLarge":null},"siteUrl":null} +]}}} diff --git a/test/fixtures/anilist/title.json b/test/fixtures/anilist/title.json new file mode 100644 index 0000000..ecd219f --- /dev/null +++ b/test/fixtures/anilist/title.json @@ -0,0 +1,11 @@ +{"data":{"Media":{"id":101,"format":"MANGA","isAdult":false, + "title":{"romaji":"Tsuki no Tabibito","english":"Moon Traveller","native":"月の旅人"}, + "synonyms":["Lunar Wanderer"],"status":"RELEASING","startDate":{"year":2019}, + "averageScore":81,"popularity":5400,"trending":37, + "genres":["Adventure","Fantasy"], + "tags":[{"name":"Travel","rank":88,"isAdult":false},{"name":"Nudity","rank":40,"isAdult":true}], + "staff":{"edges":[ + {"role":"Story & Art","node":{"id":900,"name":{"full":"Aoi Example"}}}, + {"role":"Translator","node":{"id":901,"name":{"full":"Not An Author"}}}]}, + "coverImage":{"extraLarge":"https://example.invalid/cover/101.jpg"}, + "siteUrl":"https://anilist.co/manga/101"}}}