From e6fe9aab9182e7f4a2ddb8b6958f34e6a9d3d111 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Thu, 20 Aug 2026 09:04:10 +0200 Subject: [PATCH] feat(catalog): add cheap source fingerprint cache Complete production catalog generation hashes every package twice, which is necessary for publication but wasteful when the same package tree has already produced the current catalog. Add a metadata-only cache that records package paths, sizes, nanosecond mtimes, catalog and unrar metadata, and output identity. The cache is advisory: incomplete outputs never hit, and production build validation remains the authority. Record files atomically under the ignored local cache directory, with focused tests for hits, source changes, and incomplete output. Test Plan: - `python3 -m unittest discover -s tools -p 'test_*.py'` -- passed - `git diff --cached --check` -- passed --- .gitignore | 1 + tools/catalog_source_cache.py | 202 +++++++++++++++++++++++++++++ tools/test_catalog_source_cache.py | 107 +++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 tools/catalog_source_cache.py create mode 100644 tools/test_catalog_source_cache.py diff --git a/.gitignore b/.gitignore index 1af933d..c937d08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target /.lanspread-peer-cli/ +/.lanspread/catalog-cache/ /game.db /thumbnails/ __pycache__/ diff --git a/tools/catalog_source_cache.py b/tools/catalog_source_cache.py new file mode 100644 index 0000000..40f8663 --- /dev/null +++ b/tools/catalog_source_cache.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Cheap source metadata cache for complete catalog publication.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import stat +import sys +import tempfile +from pathlib import Path +from typing import Iterable + + +CACHE_SCHEMA = 1 +CATALOG_CONTENT_INDEX = "catalog-content-index-v1.jsonl" +CATALOG_PUBLICATION_MARKER = ".lanspread-catalog-publication-in-progress" + + +def _absolute(path: Path) -> str: + return os.path.abspath(os.fspath(path)) + + +def _metadata(path: Path) -> tuple[str, int, int]: + try: + info = path.lstat() + except FileNotFoundError: + return ("missing", 0, 0) + + mode = stat.S_IFMT(info.st_mode) + kind = { + stat.S_IFDIR: "directory", + stat.S_IFREG: "file", + stat.S_IFLNK: "symlink", + }.get(mode, f"special:{mode:o}") + return (kind, info.st_size, info.st_mtime_ns) + + +def _tree_entries(root: Path) -> Iterable[tuple[str, str, int, int]]: + def visit(path: Path, relative: Path) -> Iterable[tuple[str, str, int, int]]: + kind, size, mtime_ns = _metadata(path) + yield (relative.as_posix() or ".", kind, size, mtime_ns) + if kind != "directory": + return + + with os.scandir(path) as entries: + children = sorted(entries, key=lambda entry: entry.name) + for child in children: + yield from visit(path / child.name, relative / child.name) + + yield from visit(root, Path()) + + +def _tree_fingerprint(root: Path) -> str: + digest = hashlib.sha256() + for relative, kind, size, mtime_ns in _tree_entries(root): + record = json.dumps( + [relative, kind, size, mtime_ns], + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + digest.update(record) + digest.update(b"\n") + return digest.hexdigest() + + +def _path_metadata(path: Path) -> dict[str, int | str]: + kind, size, mtime_ns = _metadata(path) + return {"kind": kind, "size": size, "mtime_ns": mtime_ns} + + +def _cache_payload( + packages_dir: Path, + catalog_db: Path, + manifests_dir: Path, + unrar: Path, +) -> dict[str, object]: + return { + "schema": CACHE_SCHEMA, + "packages_dir": _absolute(packages_dir), + "catalog_db": { + "path": _absolute(catalog_db), + **_path_metadata(catalog_db), + }, + "manifests_dir": _absolute(manifests_dir), + "unrar": { + "path": _absolute(unrar), + **_path_metadata(unrar), + }, + "packages_fingerprint": _tree_fingerprint(packages_dir), + } + + +def _output_is_ready(catalog_db: Path, manifests_dir: Path) -> bool: + db_kind, _, _ = _metadata(catalog_db) + manifests_kind, _, _ = _metadata(manifests_dir) + index_kind, _, _ = _metadata(manifests_dir / CATALOG_CONTENT_INDEX) + marker_kind, _, _ = _metadata(manifests_dir / CATALOG_PUBLICATION_MARKER) + return ( + db_kind == "file" + and manifests_kind == "directory" + and index_kind == "file" + and marker_kind == "missing" + ) + + +def cache_is_current( + stamp: Path, + packages_dir: Path, + catalog_db: Path, + manifests_dir: Path, + unrar: Path, +) -> bool: + if not _output_is_ready(catalog_db, manifests_dir): + return False + try: + cached = json.loads(stamp.read_text(encoding="utf-8")) + except (FileNotFoundError, OSError, UnicodeDecodeError, json.JSONDecodeError): + return False + return cached == _cache_payload(packages_dir, catalog_db, manifests_dir, unrar) + + +def record_cache( + stamp: Path, + packages_dir: Path, + catalog_db: Path, + manifests_dir: Path, + unrar: Path, +) -> None: + if not _output_is_ready(catalog_db, manifests_dir): + raise RuntimeError("cannot record a catalog cache before publication is complete") + stamp.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps( + _cache_payload(packages_dir, catalog_db, manifests_dir, unrar), + ensure_ascii=True, + indent=2, + sort_keys=True, + ) + with tempfile.NamedTemporaryFile( + "w", + dir=stamp.parent, + prefix=f".{stamp.name}.", + suffix=".tmp", + encoding="utf-8", + delete=False, + ) as temporary: + temporary.write(payload) + temporary.write("\n") + temporary.flush() + os.fsync(temporary.fileno()) + temporary_path = Path(temporary.name) + os.replace(temporary_path, stamp) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + for command in ("check", "record"): + subparser = subparsers.add_parser(command) + subparser.add_argument("--stamp", type=Path, required=True) + subparser.add_argument("--packages-dir", type=Path, required=True) + subparser.add_argument("--catalog-db", type=Path, required=True) + subparser.add_argument("--manifests-dir", type=Path, required=True) + subparser.add_argument("--unrar", type=Path, required=True) + return parser + + +def main(arguments: list[str] | None = None) -> int: + options = _parser().parse_args(arguments) + current = cache_is_current( + options.stamp, + options.packages_dir, + options.catalog_db, + options.manifests_dir, + options.unrar, + ) + if options.command == "check": + if current: + print(f"catalog cache hit: {options.packages_dir}") + return 0 + print(f"catalog cache miss: {options.packages_dir}") + return 1 + + record_cache( + options.stamp, + options.packages_dir, + options.catalog_db, + options.manifests_dir, + options.unrar, + ) + print(f"catalog cache recorded: {options.stamp}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, RuntimeError) as error: + print(f"catalog cache error: {error}", file=sys.stderr) + raise SystemExit(2) from error diff --git a/tools/test_catalog_source_cache.py b/tools/test_catalog_source_cache.py new file mode 100644 index 0000000..5fbcf84 --- /dev/null +++ b/tools/test_catalog_source_cache.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +import catalog_source_cache + + +class CatalogSourceCacheTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.packages = self.root / "packages" + self.packages.mkdir() + self.package = self.packages / "game" + self.package.mkdir() + (self.package / "version.ini").write_text("20260820\n", encoding="utf-8") + (self.package / "game.eti").write_bytes(b"eti") + self.catalog_db = self.root / "game.db" + self.catalog_db.write_bytes(b"catalog") + self.unrar = self.root / "unrar" + self.unrar.write_bytes(b"unrar") + self.manifests = self.root / "manifests" + self.manifests.mkdir() + (self.manifests / "catalog-content-index-v1.jsonl").write_text( + "{}\n", encoding="utf-8" + ) + self.stamp = self.root / "cache" / "catalog.json" + + def tearDown(self) -> None: + self.temporary.cleanup() + + def test_record_then_check_and_source_change(self) -> None: + self.assertFalse( + catalog_source_cache.cache_is_current( + self.stamp, + self.packages, + self.catalog_db, + self.manifests, + self.unrar, + ) + ) + catalog_source_cache.record_cache( + self.stamp, + self.packages, + self.catalog_db, + self.manifests, + self.unrar, + ) + self.assertTrue( + catalog_source_cache.cache_is_current( + self.stamp, + self.packages, + self.catalog_db, + self.manifests, + self.unrar, + ) + ) + (self.package / "payload.bin").write_bytes(b"payload") + self.assertFalse( + catalog_source_cache.cache_is_current( + self.stamp, + self.packages, + self.catalog_db, + self.manifests, + self.unrar, + ) + ) + + def test_missing_or_incomplete_output_never_hits(self) -> None: + catalog_source_cache.record_cache( + self.stamp, + self.packages, + self.catalog_db, + self.manifests, + self.unrar, + ) + (self.manifests / "catalog-content-index-v1.jsonl").unlink() + self.assertFalse( + catalog_source_cache.cache_is_current( + self.stamp, + self.packages, + self.catalog_db, + self.manifests, + self.unrar, + ) + ) + + def test_stamp_is_structured_and_sorted(self) -> None: + catalog_source_cache.record_cache( + self.stamp, + self.packages, + self.catalog_db, + self.manifests, + self.unrar, + ) + stamp = json.loads(self.stamp.read_text(encoding="utf-8")) + self.assertEqual(stamp["schema"], 1) + self.assertEqual(stamp["catalog_db"]["size"], len(b"catalog")) + self.assertEqual(stamp["unrar"]["size"], len(b"unrar")) + self.assertEqual(stamp["packages_dir"], str(self.packages.absolute())) + + +if __name__ == "__main__": + unittest.main()