Checkpoint peering identity and opt-in discovery

tools/benchmark-peer-transfer.js

@@ -0,0 +1,120 @@
+// Local-only research benchmark. Reads completed library files; writes no library state.
+const fs = require('node:fs');
+const path = require('node:path');
+const zlib = require('node:zlib');
+const crypto = require('node:crypto');
+
+const library = path.resolve(__dirname, '..', 'library');
+const series = [
+  'Solo Leveling',
+  'Gosick',
+  'The Martial God Who Regressed Back to Level 2',
+  'The Exiled Heavy Knight Knows How to Game the System',
+];
+const extensions = new Set(['.jpg', '.jpeg', '.png', '.webp', '.avif']);
+const codecs = [
+  { name: 'gzip-6', encode: (b) => zlib.gzipSync(b, { level: 6 }), decode: zlib.gunzipSync },
+  { name: 'brotli-4', encode: (b) => zlib.brotliCompressSync(b, { params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 4 } }), decode: zlib.brotliDecompressSync },
+  { name: 'zstd-3', encode: (b) => zlib.zstdCompressSync(b, { params: { [zlib.constants.ZSTD_c_compressionLevel]: 3 } }), decode: zlib.zstdDecompressSync },
+  { name: 'zstd-10', encode: (b) => zlib.zstdCompressSync(b, { params: { [zlib.constants.ZSTD_c_compressionLevel]: 10 } }), decode: zlib.zstdDecompressSync },
+];
+
+function firstCompleteChapter(manga) {
+  const root = path.join(library, manga);
+  for (const volume of fs.readdirSync(root, { withFileTypes: true }).filter((x) => x.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) {
+    const volumePath = path.join(root, volume.name);
+    for (const chapter of fs.readdirSync(volumePath, { withFileTypes: true }).filter((x) => x.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) {
+      const chapterPath = path.join(volumePath, chapter.name);
+      if (!fs.existsSync(path.join(chapterPath, 'chapter.json'))) continue;
+      const images = fs.readdirSync(chapterPath).filter((f) => extensions.has(path.extname(f).toLowerCase())).sort();
+      if (images.length) return { chapterPath, images };
+    }
+  }
+  throw new Error(`No complete chapter for ${manga}`);
+}
+
+function measure(fn) {
+  const begin = process.hrtime.bigint();
+  const result = fn();
+  return { result, ms: Number(process.hrtime.bigint() - begin) / 1e6 };
+}
+
+const chapters = series.map((manga) => ({ manga, ...firstCompleteChapter(manga) }));
+const totals = Object.fromEntries(codecs.map((codec) => [codec.name, { bytes: 0, encodeMs: 0, decodeMs: 0 }]));
+const byChapter = [];
+const seenHashes = new Set();
+let rawBytes = 0;
+let imageCount = 0;
+let duplicateBytes = 0;
+let duplicateCount = 0;
+const formats = {};
+let metadataRaw = 0;
+let metadataZstd = 0;
+const adaptive = { sampleBytes: 32768, threshold: 0.03, transmittedBytes: 0, sampleMs: 0, selectedEncodeMs: 0, selectedDecodeMs: 0, compressedImages: 0, missedUsefulImages: 0, wastedImages: 0 };
+
+for (const chapter of chapters) {
+  const row = { manga: chapter.manga, chapter: path.relative(library, chapter.chapterPath), images: chapter.images.length, rawBytes: 0, formats: {}, codecs: {} };
+  const buffers = [];
+  for (const file of chapter.images) {
+    const buffer = fs.readFileSync(path.join(chapter.chapterPath, file));
+    buffers.push(buffer);
+    imageCount++;
+    rawBytes += buffer.length;
+    row.rawBytes += buffer.length;
+    const ext = path.extname(file).toLowerCase();
+    formats[ext] = (formats[ext] || 0) + buffer.length;
+    row.formats[ext] = (row.formats[ext] || 0) + 1;
+    const hash = crypto.createHash('sha256').update(buffer).digest('hex');
+    if (seenHashes.has(hash)) { duplicateCount++; duplicateBytes += buffer.length; }
+    else seenHashes.add(hash);
+    const sample = buffer.subarray(0, Math.min(buffer.length, adaptive.sampleBytes));
+    const sampleEncoded = measure(() => codecs[2].encode(sample));
+    adaptive.sampleMs += sampleEncoded.ms;
+    const shouldCompress = sampleEncoded.result.length <= sample.length * (1 - adaptive.threshold);
+    for (const codec of codecs) {
+      const encoded = measure(() => codec.encode(buffer));
+      const decoded = measure(() => codec.decode(encoded.result));
+      if (!decoded.result.equals(buffer)) throw new Error(`Round trip failed: ${codec.name} ${file}`);
+      const total = totals[codec.name];
+      total.bytes += encoded.result.length;
+      total.encodeMs += encoded.ms;
+      total.decodeMs += decoded.ms;
+      const cell = row.codecs[codec.name] ||= { bytes: 0, encodeMs: 0, decodeMs: 0 };
+      cell.bytes += encoded.result.length;
+      cell.encodeMs += encoded.ms;
+      cell.decodeMs += decoded.ms;
+      if (codec.name === 'zstd-3') {
+        if (shouldCompress) {
+          adaptive.compressedImages++;
+          adaptive.transmittedBytes += encoded.result.length;
+          adaptive.selectedEncodeMs += encoded.ms;
+          adaptive.selectedDecodeMs += decoded.ms;
+          if (encoded.result.length >= buffer.length) adaptive.wastedImages++;
+        } else {
+          adaptive.transmittedBytes += buffer.length;
+          if (encoded.result.length <= buffer.length * 0.97) adaptive.missedUsefulImages++;
+        }
+      }
+    }
+  }
+  // Packing a chapter lets zstd exploit any repetition across page boundaries.
+  const packed = Buffer.concat(buffers);
+  const chapterEncoded = measure(() => codecs[2].encode(packed));
+  const chapterDecoded = measure(() => codecs[2].decode(chapterEncoded.result));
+  if (!chapterDecoded.result.equals(packed)) throw new Error(`Packed round trip failed: ${chapter.manga}`);
+  row.packedZstd3 = { bytes: chapterEncoded.result.length, encodeMs: chapterEncoded.ms, decodeMs: chapterDecoded.ms };
+  const metadata = fs.readFileSync(path.join(chapter.chapterPath, 'chapter.json'));
+  metadataRaw += metadata.length;
+  metadataZstd += codecs[2].encode(metadata).length;
+  byChapter.push(row);
+}
+
+console.log(JSON.stringify({
+  runtime: process.version,
+  platform: `${process.platform}/${process.arch}`,
+  sample: { imageCount, rawBytes, formats, duplicateCount, duplicateBytes, metadataRaw, metadataZstd },
+  codecs: totals,
+  adaptive,
+  chapters: byChapter,
+  method: 'One cold file read per image; encode/decode timed separately in-process using synchronous Node zlib. No network, TLS, disk-write, or browser decode time. Sizes omit protocol headers. Exact-byte round-trip verified.',
+}, null, 2));