feat(peer): prototype streamed installs
Add a streamed-install prototype that can receive archive-derived install bytes straight into local/ without first storing the peer-owned root archive payload. This is intended for low-disk clients that want to install a game but opt out of becoming a downloadable peer source for that game. The protocol gains a current-version-only StreamInstall request and framed StreamInstallFrame responses. The peer core owns the generic transport, transaction, path validation, size checks, CRC32 verification, and lifecycle state. The archive-specific work is hidden behind StreamInstallProvider so the prototype can use unrar while the final implementation can swap in a better provider without rewriting the peer command path. The receiver writes into .local.installing and only promotes to local/ after the full stream verifies. It deliberately does not write the root version.ini or archive files, so the settled local state is installed=true, downloaded=false, and availability=LocalOnly. That preserves the existing rule that local/ is not served to peers and makes streamed receivers non-sources by construction. The CLI is the only caller for now. It exposes stream-install and provides the prototype unrar implementation with unrar lt for entry metadata and unrar p for file bytes. This is simple and good enough to prove non-solid archive streaming, but it is not the production provider shape for solid archives because per-file unrar p would repeatedly decompress prefixes. The Tauri app explicitly passes stream_install_provider: None, so the GUI behavior stays unchanged until a real product path is designed. Document the production-readiness work in NEXT_STEPS.md. The main follow-up is to make the provider abstraction final-ish and replace the per-file CLI unrar provider with a one-pass archive provider, then wire a deliberate GUI low-disk mode, retry semantics, and broader failure scenarios. Test Plan: - just fmt - RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just test - python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py \ S39 S40 --build-image - RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just clippy - git diff --check - git diff --cached --check Follow-up: NEXT_STEPS.md
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the peer-cli scenarios S1-S36 through Docker."""
|
||||
"""Run the peer-cli scenarios S1-S40 through Docker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -8,6 +8,7 @@ import hashlib
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -325,6 +326,8 @@ class Runner:
|
||||
("S35", self.s35_unknown_game_filtered),
|
||||
("S36", self.s36_latest_singleton),
|
||||
("S37", self.s37_single_source_download_throughput),
|
||||
("S39", self.s39_streamed_install_local_only),
|
||||
("S40", self.s40_streamed_receiver_not_source),
|
||||
]
|
||||
|
||||
for scenario_id, scenario in scenarios:
|
||||
@@ -1060,6 +1063,114 @@ class Runner:
|
||||
f"{throughput['chunks']} chunks"
|
||||
)
|
||||
|
||||
def stream_install_cnctw(self, prefix: str) -> tuple[Peer, Peer]:
|
||||
source_dir = self.fixture_root / f"{prefix}-bravo"
|
||||
copy_game("cnctw", source_dir, version="20160128")
|
||||
source = self.peer(f"{prefix}-bravo", games_dir=source_dir)
|
||||
client = self.peer(f"{prefix}-client")
|
||||
connect_many(client, [source])
|
||||
wait_remote_game(client, "cnctw", peer_count=1)
|
||||
waiter = LineWaiter(len(client.output))
|
||||
client.send({"cmd": "stream-install", "game_id": "cnctw"})
|
||||
client.wait_for(
|
||||
event_is("got-game-files", "cnctw"),
|
||||
timeout=20,
|
||||
description="got cnctw files",
|
||||
waiter=waiter,
|
||||
)
|
||||
client.wait_for(
|
||||
event_is("download-begin", "cnctw"),
|
||||
timeout=20,
|
||||
description="stream begin cnctw",
|
||||
waiter=waiter,
|
||||
)
|
||||
client.wait_for(
|
||||
event_is("download-finished", "cnctw"),
|
||||
timeout=60,
|
||||
description="stream finish cnctw",
|
||||
waiter=waiter,
|
||||
)
|
||||
client.wait_for(
|
||||
event_is("install-finished", "cnctw"),
|
||||
timeout=30,
|
||||
description="stream install cnctw",
|
||||
waiter=waiter,
|
||||
)
|
||||
return source, client
|
||||
|
||||
def s39_streamed_install_local_only(self) -> str:
|
||||
source, client = self.stream_install_cnctw("s39")
|
||||
game = wait_local_game(client, "cnctw", downloaded=False, installed=True)
|
||||
assert_game_state(
|
||||
game,
|
||||
downloaded=False,
|
||||
installed=True,
|
||||
availability="LocalOnly",
|
||||
)
|
||||
|
||||
game_root = client.host_games_dir / "cnctw"
|
||||
assert_not_exists(game_root / "version.ini")
|
||||
assert_not_exists(game_root / "cnctw.eti")
|
||||
|
||||
expected = {
|
||||
"bin/cnctw-payload.bin": unrar_entry_sha256(
|
||||
source, "cnctw", "bin/cnctw-payload.bin"
|
||||
),
|
||||
"data/cnctw-assets.dat": unrar_entry_sha256(
|
||||
source, "cnctw", "data/cnctw-assets.dat"
|
||||
),
|
||||
}
|
||||
actual = {
|
||||
rel: sha256_file(game_root / "local" / rel)
|
||||
for rel in expected
|
||||
}
|
||||
if actual != expected:
|
||||
raise ScenarioError(f"streamed local payload hashes mismatched: {actual} != {expected}")
|
||||
|
||||
streamed_bytes = sum(
|
||||
int(item.get("data", {}).get("length", 0))
|
||||
for item in client.output
|
||||
if item.get("type") == "event"
|
||||
and item.get("event") == "download-chunk-finished"
|
||||
and item.get("data", {}).get("game_id") == "cnctw"
|
||||
)
|
||||
expected_bytes = 3 * 1024 * 1024
|
||||
if streamed_bytes != expected_bytes:
|
||||
raise ScenarioError(
|
||||
f"streamed byte count mismatch: {streamed_bytes} != {expected_bytes}"
|
||||
)
|
||||
|
||||
return (
|
||||
"cnctw streamed into local/ only; root archive and version.ini absent; "
|
||||
f"payload hashes={actual}"
|
||||
)
|
||||
|
||||
def s40_streamed_receiver_not_source(self) -> str:
|
||||
_source, receiver = self.stream_install_cnctw("s40")
|
||||
observer = self.peer("s40-observer")
|
||||
connect_many(observer, [receiver])
|
||||
receiver_snapshot = wait_peer_has_game(observer, receiver.peer_id, "cnctw")
|
||||
summary = next(
|
||||
game
|
||||
for game in receiver_snapshot.get("games", [])
|
||||
if game.get("id") == "cnctw"
|
||||
)
|
||||
if summary.get("availability") != "LocalOnly" or summary.get("downloaded"):
|
||||
raise ScenarioError(f"receiver did not advertise cnctw as local-only: {summary}")
|
||||
|
||||
wait_remote_absent(observer, "cnctw", timeout=5)
|
||||
err = observer.send(
|
||||
{"cmd": "download", "game_id": "cnctw", "install": False},
|
||||
expect_error=True,
|
||||
)
|
||||
if "no peers have game cnctw" not in err["error"]:
|
||||
raise ScenarioError(f"unexpected local-only download error: {err}")
|
||||
assert_not_exists(observer.host_games_dir / "cnctw")
|
||||
return (
|
||||
"observer saw receiver's local-only cnctw snapshot, but remote aggregation hid it "
|
||||
f"and download errored '{err['error']}'"
|
||||
)
|
||||
|
||||
|
||||
def run(command: list[str], description: str) -> subprocess.CompletedProcess[str]:
|
||||
result = subprocess.run(
|
||||
@@ -1177,6 +1288,25 @@ def create_large_sparse_game(root: Path, *, size: int) -> None:
|
||||
handle.truncate(size)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
hasher = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def unrar_entry_sha256(peer: Peer, game_id: str, relative_path: str) -> str:
|
||||
command = (
|
||||
f"unrar p -inul /games/{shlex.quote(game_id)}/{shlex.quote(game_id)}.eti "
|
||||
f"{shlex.quote(relative_path)} | sha256sum"
|
||||
)
|
||||
output = peer.docker_exec("sh", "-c", command).stdout.strip()
|
||||
if not output:
|
||||
raise ScenarioError(f"empty sha256 output for {game_id}:{relative_path}")
|
||||
return output.split()[0]
|
||||
|
||||
|
||||
def format_bytes(size: int) -> str:
|
||||
return f"{size / 1024 / 1024 / 1024:.2f} GiB"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user