diff --git a/discovery/anilist.js b/discovery/anilist.js index 184dcc2..cad2d02 100644 --- a/discovery/anilist.js +++ b/discovery/anilist.js @@ -29,8 +29,24 @@ function createLimiter({ perMinute = 25, now = Date.now, sleep = ms => new Promi return { wait }; } +const invalid = message => new AniListSkip('invalid', `AniList response is malformed: ${message}`); +const isObject = value => !!value && typeof value === 'object' && !Array.isArray(value); +const optionalArray = (value, what) => { if (value != null && !Array.isArray(value)) throw invalid(what); }; + +// Malformed input is a Skip('invalid'), never a plain TypeError from deep inside the mapping. +function assertShape(media) { + if (media.title != null && !isObject(media.title)) throw invalid('title'); + for (const key of ['synonyms', 'genres', 'tags']) optionalArray(media[key], key); + if (media.staff != null && !isObject(media.staff)) throw invalid('staff'); + optionalArray(media.staff?.edges, 'staff edges'); + if ((media.tags || []).some(tag => !isObject(tag))) throw invalid('tags'); + if ((media.staff?.edges || []).some(edge => !isObject(edge) || (edge.node != null && !isObject(edge.node)))) throw invalid('staff edges'); +} + function normalizeMedia(media) { - if (!media || (media.format !== 'MANGA' && media.format !== 'ONE_SHOT')) return null; + if (!isObject(media) || !Number.isInteger(media.id)) throw invalid('media without integer id'); + if (media.format !== 'MANGA' && media.format !== 'ONE_SHOT') return null; + assertShape(media); const title = media.title || {}; const names = [title.english || title.romaji || title.native, title.romaji, title.native, ...(media.synonyms || [])].filter(Boolean); return { @@ -69,15 +85,14 @@ const QUERIES = { creator: `query ($id: Int) { Staff(id: $id) { staffMedia(type: MANGA, perPage: 25, sort: POPULARITY_DESC) { edges { staffRole node { ...MediaFields } } } } } ${MEDIA_FIELDS}`, tagTopGenre: `query ($genre_in: [String]) { Page(perPage: 25) { - media(type: MANGA, genre_in: $genre_in, popularity_greater: 1000, sort: POPULARITY_DESC) { ...MediaFields } } } ${MEDIA_FIELDS}`, + media(type: MANGA, genre_in: $genre_in, popularity_greater: 1000, sort: SCORE_DESC) { ...MediaFields } } } ${MEDIA_FIELDS}`, tagTopTag: `query ($tag_in: [String]) { Page(perPage: 25) { - media(type: MANGA, tag_in: $tag_in, popularity_greater: 1000, sort: POPULARITY_DESC) { ...MediaFields } } } ${MEDIA_FIELDS}`, + media(type: MANGA, tag_in: $tag_in, popularity_greater: 1000, sort: SCORE_DESC) { ...MediaFields } } } ${MEDIA_FIELDS}`, fresh: `query ($startDate_greater: FuzzyDateInt, $perPage: Int) { Page(perPage: $perPage) { media(type: MANGA, status: RELEASING, startDate_greater: $startDate_greater, sort: TRENDING_DESC) { ...MediaFields } } } ${MEDIA_FIELDS}`, }; const isTimeout = error => error && (error.name === 'TimeoutError' || error.name === 'AbortError'); -const hasId = media => media && Number.isInteger(media.id); const need = (value, check = Array.isArray) => { if (!check(value)) throw new AniListSkip('invalid', 'AniList response is missing data'); return value; }; function createClient({ endpoint = process.env.DISCOVERY_ANILIST_URL || 'https://graphql.anilist.co', limiter = createLimiter(), @@ -88,57 +103,58 @@ function createClient({ endpoint = process.env.DISCOVERY_ANILIST_URL || 'https:/ try { response = await fetchImpl(endpoint, { method: 'POST', headers: { 'content-type': 'application/json', 'user-agent': USER_AGENT }, - body: JSON.stringify({ query, variables }), signal: AbortSignal.timeout(timeoutMs) }); + body: JSON.stringify({ query, variables }), signal: AbortSignal.timeout(timeoutMs), + redirect: 'error' }); // never follow a redirect to another host if (response.status === 429) throw new AniListStop('rate-limited'); if (response.status === 401 || response.status === 403) throw new AniListStop('forbidden'); - if (response.status >= 500) throw new AniListSkip('upstream', `AniList HTTP ${response.status}`); text = await response.text(); } catch (error) { if (error instanceof AniListStop || error instanceof AniListSkip) throw error; if (isTimeout(error)) throw new AniListSkip('timeout'); throw new AniListSkip('upstream', `AniList request failed: ${error.message}`); } - // An HTML or non-JSON page means a bot-check in front of the API; never work around it. + // An HTML or non-JSON page means a bot-check in front of the API, even on a 5xx; never work around it. if (/text\/html/i.test(response.headers.get('content-type') || '')) throw new AniListStop('challenge'); let body; try { body = JSON.parse(text); } catch { throw new AniListStop('challenge'); } + if (response.status >= 500) throw new AniListSkip('upstream', `AniList HTTP ${response.status}`); if (!response.ok || !body || typeof body !== 'object') throw new AniListSkip('invalid', `AniList HTTP ${response.status}`); return body.data || {}; } - const records = (list, pick) => need(list).filter(hasId).map(item => ({ item, record: normalizeMedia(pick(item)) })).filter(entry => entry.record); - const mediaOf = data => { need(data.Media, value => value && typeof value === 'object'); return data.Media; }; + // Every item is normalized (and so validated) before any is dropped: one malformed item invalidates the response. + const mediaOf = data => need(data.Media, isObject); + const entries = (list, pick) => need(list).map(item => { + if (!isObject(item)) throw invalid('list item'); + const media = pick(item); + return { item, media, record: normalizeMedia(media) }; + }).filter(entry => entry.record); return { async title(id) { - const media = mediaOf(await post(QUERIES.title, { id })); - if (!hasId(media)) throw new AniListSkip('invalid'); - return normalizeMedia(media); + return normalizeMedia(mediaOf(await post(QUERIES.title, { id }))); }, async recs(id) { - const nodes = need(mediaOf(await post(QUERIES.recs, { id })).recommendations?.nodes); - return nodes.filter(node => node.rating > 0 && hasId(node.mediaRecommendation)) - .map(node => ({ anilistId: node.mediaRecommendation.id, rating: node.rating, record: normalizeMedia(node.mediaRecommendation) })) - .filter(entry => entry.record); + const nodes = mediaOf(await post(QUERIES.recs, { id })).recommendations?.nodes; + return entries(nodes, node => node.mediaRecommendation).filter(({ item }) => item.rating > 0) + .map(({ item, record }) => ({ anilistId: record.anilistId, rating: item.rating, record })); }, async creator(staffId) { const data = await post(QUERIES.creator, { id: staffId }); - const edges = need(need(data.Staff, value => value && typeof value === 'object').staffMedia?.edges); - return edges.filter(edge => hasId(edge.node)) - .map(edge => ({ anilistId: edge.node.id, role: edge.staffRole, record: normalizeMedia(edge.node) })) - .filter(entry => entry.record); + return entries(need(data.Staff, isObject).staffMedia?.edges, edge => edge.node) + .map(({ item, record }) => ({ anilistId: record.anilistId, role: item.staffRole, record })); }, async tagTop({ name, kind }) { const genre = kind === 'genre'; const data = await post(genre ? QUERIES.tagTopGenre : QUERIES.tagTopTag, genre ? { genre_in: [name] } : { tag_in: [name] }); - return records(data.Page?.media, media => media).map(({ item, record }) => ({ anilistId: item.id, record })); + return entries(data.Page?.media, media => media).map(({ record }) => ({ anilistId: record.anilistId, record })); }, async fresh() { const date = today(); // AniList FuzzyDateInt: YYYYMMDD, two years back. const startDate = (date.getUTCFullYear() - 2) * 10000 + (date.getUTCMonth() + 1) * 100 + date.getUTCDate(); const data = await post(QUERIES.fresh, { startDate_greater: startDate, perPage: 50 }); - return records(data.Page?.media, media => media).map(({ item, record }) => ({ anilistId: item.id, record })); + return entries(data.Page?.media, media => media).map(({ record }) => ({ anilistId: record.anilistId, record })); }, }; } diff --git a/test/discovery-anilist.test.js b/test/discovery-anilist.test.js index fc98aac..7a88776 100644 --- a/test/discovery-anilist.test.js +++ b/test/discovery-anilist.test.js @@ -51,6 +51,46 @@ test('recs keeps positive ratings and manga formats', async () => { assert.equal(out[0].record.title, 'Sky Town'); }); +test('recs query follows spec sort and page size', async () => { + const { client, calls } = stub(() => json(fixture('recs'))); + await client.recs(101); + assert.match(calls[0].body.query, /recommendations\(perPage: 25, sort: RATING_DESC\)/); +}); + +test('creator query follows spec sort and page size', async () => { + const { client, calls } = stub(() => json(fixture('creator'))); + await client.creator(900); + assert.match(calls[0].body.query, /staffMedia\(type: MANGA, perPage: 25, sort: POPULARITY_DESC\)/); +}); + +test('title on a novel resolves to null', async () => { + assert.equal(await clientFor(() => json(fixture('novel'))).title(777), null); +}); + +test('requests refuse redirects', async () => { + const { client, calls } = stub(() => json(fixture('title'))); + await client.title(101); + assert.equal(calls[0].init.redirect, 'error'); + const redirected = () => { throw new TypeError('fetch failed: unexpected redirect'); }; + await assert.rejects(clientFor(redirected).title(1), e => e instanceof AniListSkip && e.reason === 'upstream'); +}); + +test('one malformed item invalidates a list response', async () => { + const invalid = e => e instanceof AniListSkip && e.reason === 'invalid'; + const recs = fixture('recs'); + recs.data.Media.recommendations.nodes.push(null); + await assert.rejects(clientFor(() => json(recs)).recs(1), invalid); + const tagtop = fixture('tagtop'); + tagtop.data.Page.media[0].id = 'x'; + await assert.rejects(clientFor(() => json(tagtop)).tagTop({ name: 'Mystery', kind: 'genre' }), invalid); + for (const patch of [{ genres: 'Drama' }, { tags: [null] }, { title: 'x' }, { staff: { edges: {} } }, { staff: { edges: [null] } }]) { + const fresh = fixture('fresh'); + Object.assign(fresh.data.Page.media[0], patch); + await assert.rejects(clientFor(() => json(fresh)).fresh(), invalid, JSON.stringify(patch)); + } + await assert.rejects(async () => normalizeMedia({ id: 1, format: 'MANGA', title: 5 }), invalid); +}); + test('creator returns roles and drops novels', async () => { const out = await clientFor(() => json(fixture('creator'))).creator(900); assert.deepEqual(out.map(e => [e.anilistId, e.role]), [[301, 'Story & Art']]); @@ -70,10 +110,12 @@ test('tagTop uses genre_in for genres and tag_in for tags', async () => { const out = await g.client.tagTop({ name: 'Mystery', kind: 'genre' }); assert.deepEqual(g.calls[0].body.variables.genre_in, ['Mystery']); assert.equal(g.calls[0].body.variables.tag_in, undefined); - assert.match(g.calls[0].body.query, /popularity_greater: 1000/); + assert.match(g.calls[0].body.query, /genre_in: \$genre_in, popularity_greater: 1000, sort: SCORE_DESC/); + assert.match(g.calls[0].body.query, /Page\(perPage: 25\)/); assert.deepEqual(out.map(e => e.anilistId), [401]); const t = stub(() => json(fixture('tagtop'))); await t.client.tagTop({ name: 'Detective', kind: 'tag' }); + assert.match(t.calls[0].body.query, /tag_in: \$tag_in, popularity_greater: 1000, sort: SCORE_DESC/); assert.deepEqual(t.calls[0].body.variables.tag_in, ['Detective']); assert.equal(t.calls[0].body.variables.genre_in, undefined); }); @@ -90,6 +132,11 @@ test('429 stops, 403 stops, html body is a challenge', async () => { test('5xx and timeout skip', async () => { await assert.rejects(clientFor(() => json({}, 503)).title(1), e => e instanceof AniListSkip && e.reason === 'upstream'); + // A challenge page served with a 5xx is still a challenge, not an outage. + const html503 = () => new Response('just a moment', { status: 503, headers: { 'content-type': 'text/html' } }); + await assert.rejects(clientFor(html503).title(1), e => e instanceof AniListStop && e.reason === 'challenge'); + const text502 = () => new Response('Bad Gateway', { status: 502 }); + await assert.rejects(clientFor(text502).title(1), e => e instanceof AniListStop && e.reason === 'challenge'); const timeout = () => { throw Object.assign(new Error('timed out'), { name: 'TimeoutError' }); }; await assert.rejects(clientFor(timeout).title(1), e => e instanceof AniListSkip && e.reason === 'timeout'); const abort = () => { throw Object.assign(new Error('aborted'), { name: 'AbortError' }); };