Consolidate MangaCircus app, scraper, and Compose baseline
This commit is contained in:
Maashkinoozhe committed
1 parent
cfb2d79103
commit
eef4ffa07c
103 files changed
+9316
-172
No files matched your search
@@ -0,0 +1,300 @@
|
||||
'use strict';
|
||||
|
||||
const INPUT_TYPES = ['mouse', 'touch', 'keyboard'];
|
||||
const MAX_RECENT_SESSIONS = 8;
|
||||
|
||||
const whole = (value, max = Number.MAX_SAFE_INTEGER) => Math.min(max, Math.max(0, Math.round(Number(value) || 0)));
|
||||
|
||||
function rangeSet(ranges, pageCount = Number.MAX_SAFE_INTEGER) {
|
||||
const out = new Set();
|
||||
for (const pair of Array.isArray(ranges) ? ranges : []) {
|
||||
if (!Array.isArray(pair) || pair.length < 2) continue;
|
||||
const start = whole(pair[0], pageCount - 1), end = whole(pair[1], pageCount - 1);
|
||||
for (let i = Math.min(start, end); i <= Math.max(start, end); i++) out.add(i);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function compactRanges(values) {
|
||||
const sorted = [...new Set(values)].filter(Number.isInteger).sort((a, b) => a - b);
|
||||
const ranges = [];
|
||||
for (const value of sorted) {
|
||||
const last = ranges[ranges.length - 1];
|
||||
if (last && value <= last[1] + 1) last[1] = Math.max(last[1], value);
|
||||
else ranges.push([value, value]);
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
function eligiblePages(version, settings) {
|
||||
const eligible = [];
|
||||
for (let i = 0; i < version.pages.length; i++) {
|
||||
const tags = version.pages[i].t || [];
|
||||
if (settings.skipExtras && tags.includes('extra')) continue;
|
||||
if (settings.skipCovers && tags.includes('cover')) continue;
|
||||
eligible.push(i);
|
||||
}
|
||||
return eligible.length ? eligible : version.pages.map((_, i) => i);
|
||||
}
|
||||
|
||||
function emptyRecord() {
|
||||
return {
|
||||
schema: 1,
|
||||
totalActiveMs: 0,
|
||||
firstCompleteMs: null,
|
||||
lastCompleteMs: null,
|
||||
loadingMs: 0,
|
||||
sessions: 0,
|
||||
completedPasses: 0,
|
||||
inputMs: { mouse: 0, touch: 0, keyboard: 0 },
|
||||
fastBursts: 0,
|
||||
rejectedPageViews: 0,
|
||||
maxObservedPps: 0,
|
||||
currentPass: null,
|
||||
recentSessions: [],
|
||||
updatedAt: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeRecord(record) {
|
||||
const base = emptyRecord(), out = record && typeof record === 'object' ? record : {};
|
||||
for (const [key, value] of Object.entries(base)) if (out[key] === undefined) out[key] = value;
|
||||
out.inputMs = { ...base.inputMs, ...(out.inputMs || {}) };
|
||||
out.recentSessions = Array.isArray(out.recentSessions) ? out.recentSessions : [];
|
||||
return out;
|
||||
}
|
||||
|
||||
function publicSummary(record, version, eligible) {
|
||||
const r = normalizeRecord(record || emptyRecord());
|
||||
const pass = r.currentPass && r.currentPass.version === version ? r.currentPass : null;
|
||||
const allowed = new Set(eligible);
|
||||
const seen = pass ? [...rangeSet(pass.seenRanges)].filter(x => allowed.has(x)) : [];
|
||||
const predominantInput = INPUT_TYPES.reduce((best, key) => r.inputMs[key] > r.inputMs[best] ? key : best, 'mouse');
|
||||
const hasInput = INPUT_TYPES.some(key => r.inputMs[key] > 0);
|
||||
return {
|
||||
schema: 1,
|
||||
totalActiveMs: whole(r.totalActiveMs),
|
||||
firstCompleteMs: r.firstCompleteMs == null ? null : whole(r.firstCompleteMs),
|
||||
lastCompleteMs: r.lastCompleteMs == null ? null : whole(r.lastCompleteMs),
|
||||
loadingMs: whole(r.loadingMs),
|
||||
sessions: whole(r.sessions),
|
||||
completedPasses: whole(r.completedPasses),
|
||||
inputMs: Object.fromEntries(INPUT_TYPES.map(key => [key, whole(r.inputMs[key])])),
|
||||
predominantInput: hasInput ? predominantInput : null,
|
||||
fastBursts: whole(r.fastBursts),
|
||||
rejectedPageViews: whole(r.rejectedPageViews),
|
||||
maxObservedPps: whole(r.maxObservedPps),
|
||||
eligiblePages: eligible.length,
|
||||
currentPass: {
|
||||
version,
|
||||
activeMs: whole(pass && pass.activeMs),
|
||||
seenRanges: compactRanges(seen),
|
||||
seenPages: seen.length,
|
||||
complete: !!(pass && pass.completedAt),
|
||||
startedAt: pass ? whole(pass.startedAt) : null,
|
||||
protocol: pass?.protocol || 1,
|
||||
requiredPages: Math.ceil(eligible.length * (pass?.protocol === 2 ? 0.6 : 1)),
|
||||
},
|
||||
updatedAt: whole(r.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function chapterInsight(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
const r = normalizeRecord(record);
|
||||
const totalActiveMs = whole(r.totalActiveMs);
|
||||
if (!totalActiveMs && !whole(r.completedPasses) && (!whole(r.updatedAt) || r.currentPass?.protocol === 2)) return null;
|
||||
return {
|
||||
totalActiveMs,
|
||||
firstCompleteMs: r.firstCompleteMs == null ? null : whole(r.firstCompleteMs),
|
||||
lastCompleteMs: r.lastCompleteMs == null ? null : whole(r.lastCompleteMs),
|
||||
completedPasses: whole(r.completedPasses),
|
||||
currentPassMs: whole(r.currentPass && r.currentPass.activeMs),
|
||||
currentPassComplete: !!(r.currentPass && r.currentPass.completedAt),
|
||||
updatedAt: whole(r.qualifiedUpdatedAt || r.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function median(values) {
|
||||
const sorted = values.filter(value => Number.isFinite(value) && value > 0).sort((a, b) => a - b);
|
||||
if (!sorted.length) return null;
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 ? whole(sorted[middle]) : whole((sorted[middle - 1] + sorted[middle]) / 2);
|
||||
}
|
||||
|
||||
function mangaInsight(records, chapterIds) {
|
||||
const source = records && typeof records === 'object' ? records : {};
|
||||
const ids = Array.isArray(chapterIds) ? chapterIds : Object.keys(source);
|
||||
const chapters = ids.map(id => ({ id, reading: chapterInsight(source[id]) })).filter(item => item.reading);
|
||||
const latest = chapters.reduce((best, item) => !best || item.reading.updatedAt > best.reading.updatedAt ? item : best, null);
|
||||
const completed = chapters.filter(item => item.reading.completedPasses > 0);
|
||||
return {
|
||||
activeMs: chapters.reduce((sum, item) => sum + item.reading.totalActiveMs, 0),
|
||||
timedChapters: chapters.filter(item => item.reading.totalActiveMs > 0).length,
|
||||
completedChapters: completed.length,
|
||||
completedPasses: completed.reduce((sum, item) => sum + item.reading.completedPasses, 0),
|
||||
rereads: completed.reduce((sum, item) => sum + Math.max(0, item.reading.completedPasses - 1), 0),
|
||||
typicalChapterMs: median(completed.map(item => item.reading.lastCompleteMs)),
|
||||
lastChapterId: latest ? latest.id : null,
|
||||
updatedAt: latest ? latest.reading.updatedAt : 0,
|
||||
};
|
||||
}
|
||||
|
||||
function libraryInsight(mangas) {
|
||||
const tracked = (Array.isArray(mangas) ? mangas : []).filter(item => item && item.reading && item.reading.activeMs > 0);
|
||||
const most = tracked.reduce((best, item) => !best || item.reading.activeMs > best.reading.activeMs ? item : best, null);
|
||||
const latest = tracked.reduce((best, item) => !best || item.reading.updatedAt > best.reading.updatedAt ? item : best, null);
|
||||
return {
|
||||
activeMs: tracked.reduce((sum, item) => sum + item.reading.activeMs, 0),
|
||||
timedChapters: tracked.reduce((sum, item) => sum + item.reading.timedChapters, 0),
|
||||
completedPasses: tracked.reduce((sum, item) => sum + item.reading.completedPasses, 0),
|
||||
rereads: tracked.reduce((sum, item) => sum + item.reading.rereads, 0),
|
||||
timedMangas: tracked.length,
|
||||
mostTimeMangaId: most ? most.id : null,
|
||||
lastMangaId: latest ? latest.id : null,
|
||||
lastChapterId: latest ? latest.reading.lastChapterId : null,
|
||||
updatedAt: latest ? latest.reading.updatedAt : 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Cumulative visits tolerate retry and an exit beacon overtaking a regular pulse.
|
||||
// Coverage belongs to THIS visit, so an old read cannot qualify a picture lookup.
|
||||
// Legacy totals are preserved; unqualified visit time is a draft, not a statistic.
|
||||
function applyVisit(record, pulse, { version, pageCount, eligible, now = Date.now() } = {}) {
|
||||
const r = normalizeRecord(record), session = String(pulse.session || ''), seq = whole(pulse.seq, 1e9);
|
||||
let receipt = r.recentSessions.find(item => item.id === session);
|
||||
if (receipt && seq <= whole(receipt.seq)) return { duplicate: true, record: r };
|
||||
if (!receipt) {
|
||||
if (!r.qualifiedUpdatedAt && r.totalActiveMs) r.qualifiedUpdatedAt = r.updatedAt;
|
||||
receipt = { id: session, seq: 0, at: now, pass: {
|
||||
protocol: 2, visit: session, version, pageCount, seenRanges: [], activeMs: 0,
|
||||
loadingMs: 0, inputMs: { mouse: 0, touch: 0, keyboard: 0 }, creditedMs: 0,
|
||||
startedAt: now, completedAt: null,
|
||||
} };
|
||||
r.recentSessions.push(receipt);
|
||||
r.currentPass = receipt.pass;
|
||||
}
|
||||
const pass = receipt.pass;
|
||||
if (!pass || pass.version !== version) return { duplicate: true, record: r };
|
||||
// Wall-time sanity, not a speed test or the legacy twelve-second packet cap.
|
||||
const budget = whole(pulse.elapsedMs, 24 * 60 * 60 * 1000) + 1500;
|
||||
const requestedActive = whole(pulse.activeMs, budget), requestedLoading = whole(pulse.loadingMs, budget);
|
||||
const scale = requestedActive + requestedLoading > budget ? budget / (requestedActive + requestedLoading) : 1;
|
||||
const active = Math.max(pass.activeMs, Math.round(requestedActive * scale));
|
||||
const loading = Math.max(pass.loadingMs, Math.round(requestedLoading * scale));
|
||||
r.loadingMs += loading - pass.loadingMs;
|
||||
pass.activeMs = active; pass.loadingMs = loading;
|
||||
const input = Object.fromEntries(INPUT_TYPES.map(key => [key, whole(pulse.inputMs?.[key], active)]));
|
||||
const inputTotal = INPUT_TYPES.reduce((sum, key) => sum + input[key], 0);
|
||||
const inputScale = inputTotal > active ? active / inputTotal : 1;
|
||||
for (const key of INPUT_TYPES) pass.inputMs[key] = Math.max(pass.inputMs[key], Math.round(input[key] * inputScale));
|
||||
const allowed = new Set(eligible), seen = rangeSet(pass.seenRanges, pageCount);
|
||||
for (const page of Array.isArray(pulse.seen) ? pulse.seen.slice(0, pageCount) : []) {
|
||||
if (Number.isInteger(page) && allowed.has(page)) seen.add(page);
|
||||
}
|
||||
pass.seenRanges = compactRanges(seen);
|
||||
const covered = eligible.filter(page => seen.has(page)).length;
|
||||
if (!pass.completedAt && eligible.length && covered >= Math.ceil(eligible.length * 0.6) && active > 0) {
|
||||
pass.completedAt = now;
|
||||
pass.ordinal = ++r.completedPasses;
|
||||
r.sessions++;
|
||||
}
|
||||
if (pass.completedAt) {
|
||||
r.totalActiveMs += active - pass.creditedMs;
|
||||
pass.creditedMs = active;
|
||||
pass.creditedInput = pass.creditedInput || { mouse: 0, touch: 0, keyboard: 0 };
|
||||
for (const key of INPUT_TYPES) {
|
||||
r.inputMs[key] += pass.inputMs[key] - pass.creditedInput[key];
|
||||
pass.creditedInput[key] = pass.inputMs[key];
|
||||
}
|
||||
// Duration keeps growing through the last page and departure, not just 60%.
|
||||
if (pass.ordinal === 1) r.firstCompleteMs = active;
|
||||
if (pass.ordinal === r.completedPasses) r.lastCompleteMs = active;
|
||||
r.qualifiedUpdatedAt = now;
|
||||
}
|
||||
pass.closed = !!pulse.final;
|
||||
receipt.seq = seq; receipt.at = now;
|
||||
if (r.currentPass?.visit === session) r.currentPass = pass;
|
||||
r.recentSessions.sort((a, b) => whole(b.at) - whole(a.at));
|
||||
r.recentSessions.splice(MAX_RECENT_SESSIONS);
|
||||
r.updatedAt = now;
|
||||
return { duplicate: false, record: r };
|
||||
}
|
||||
|
||||
function applyPulse(record, pulse, { version, pageCount, eligible, now = Date.now(), maxDeltaMs = 12000 } = {}) {
|
||||
const r = normalizeRecord(record);
|
||||
const session = String(pulse.session || '');
|
||||
const seq = whole(pulse.seq, 1e9);
|
||||
let receipt = r.recentSessions.find(item => item.id === session);
|
||||
if (receipt && seq <= whole(receipt.seq, 1e9)) return { duplicate: true, record: r };
|
||||
|
||||
const newSession = !receipt;
|
||||
if (!receipt) {
|
||||
receipt = { id: session, seq: 0, at: now, counted: false };
|
||||
r.recentSessions.push(receipt);
|
||||
}
|
||||
|
||||
const elapsed = newSession ? maxDeltaMs : Math.max(0, now - whole(receipt.at));
|
||||
const allowedDelta = Math.min(maxDeltaMs, elapsed + 1500);
|
||||
const requestedActive = whole(pulse.activeMs, maxDeltaMs * 2);
|
||||
const requestedLoading = whole(pulse.loadingMs, maxDeltaMs * 2);
|
||||
const requestedTotal = requestedActive + requestedLoading;
|
||||
const scale = requestedTotal > allowedDelta && requestedTotal ? allowedDelta / requestedTotal : 1;
|
||||
const activeMs = Math.round(requestedActive * scale);
|
||||
const loadingMs = Math.round(requestedLoading * scale);
|
||||
|
||||
receipt.seq = seq;
|
||||
receipt.at = now;
|
||||
if (!receipt.counted && activeMs > 0) { receipt.counted = true; r.sessions++; }
|
||||
r.recentSessions.sort((a, b) => whole(b.at) - whole(a.at));
|
||||
r.recentSessions.splice(MAX_RECENT_SESSIONS);
|
||||
|
||||
const prior = r.currentPass;
|
||||
if (!prior || prior.version !== version || (prior.completedAt && newSession)) {
|
||||
r.currentPass = {
|
||||
version,
|
||||
pageCount,
|
||||
seenRanges: [],
|
||||
activeMs: 0,
|
||||
startedAt: now,
|
||||
completedAt: null,
|
||||
};
|
||||
}
|
||||
const pass = r.currentPass;
|
||||
pass.pageCount = pageCount;
|
||||
pass.activeMs = whole(pass.activeMs) + activeMs;
|
||||
|
||||
r.totalActiveMs = whole(r.totalActiveMs) + activeMs;
|
||||
r.loadingMs = whole(r.loadingMs) + loadingMs;
|
||||
const rawInput = pulse.inputMs && typeof pulse.inputMs === 'object' ? pulse.inputMs : {};
|
||||
const requestedInput = Object.fromEntries(INPUT_TYPES.map(key => [key, whole(rawInput[key], maxDeltaMs * 2)]));
|
||||
const inputTotal = INPUT_TYPES.reduce((sum, key) => sum + requestedInput[key], 0);
|
||||
const inputScale = inputTotal > activeMs && inputTotal ? activeMs / inputTotal : 1;
|
||||
for (const key of INPUT_TYPES) r.inputMs[key] = whole(r.inputMs[key]) + Math.round(requestedInput[key] * inputScale);
|
||||
|
||||
const allowedPages = new Set(eligible);
|
||||
const seen = rangeSet(pass.seenRanges, pageCount);
|
||||
for (const raw of Array.isArray(pulse.seen) ? pulse.seen.slice(0, pageCount) : []) {
|
||||
const page = Number(raw);
|
||||
if (Number.isInteger(page) && allowedPages.has(page)) seen.add(page);
|
||||
}
|
||||
pass.seenRanges = compactRanges([...seen]);
|
||||
|
||||
r.fastBursts = whole(r.fastBursts) + whole(pulse.fastBursts, 100);
|
||||
r.rejectedPageViews = whole(r.rejectedPageViews) + whole(pulse.rejectedPageViews, pageCount * 2);
|
||||
r.maxObservedPps = Math.max(whole(r.maxObservedPps), whole(pulse.maxPps, 1000));
|
||||
|
||||
if (!pass.completedAt && eligible.length && eligible.every(page => seen.has(page))) {
|
||||
pass.completedAt = now;
|
||||
r.completedPasses = whole(r.completedPasses) + 1;
|
||||
r.lastCompleteMs = pass.activeMs;
|
||||
if (r.firstCompleteMs == null) r.firstCompleteMs = pass.activeMs;
|
||||
}
|
||||
r.updatedAt = now;
|
||||
return { duplicate: false, record: r, activeMs, loadingMs };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
INPUT_TYPES, applyPulse, applyVisit, chapterInsight, compactRanges, eligiblePages, emptyRecord,
|
||||
libraryInsight, mangaInsight, publicSummary, rangeSet,
|
||||
};
|
||||
Reference in new issue
Block a user