feat(peer): prototype streamed installs

Add a streamed-install prototype that can receive archive-derived install bytes
straight into local/ without first storing the peer-owned root archive payload.
This is intended for low-disk clients that want to install a game but opt out of
becoming a downloadable peer source for that game.

The protocol gains a current-version-only StreamInstall request and framed
StreamInstallFrame responses. The peer core owns the generic transport,
transaction, path validation, size checks, CRC32 verification, and lifecycle
state. The archive-specific work is hidden behind StreamInstallProvider so the
prototype can use unrar while the final implementation can swap in a better
provider without rewriting the peer command path.

The receiver writes into .local.installing and only promotes to local/ after the
full stream verifies. It deliberately does not write the root version.ini or
archive files, so the settled local state is installed=true, downloaded=false,
and availability=LocalOnly. That preserves the existing rule that local/ is not
served to peers and makes streamed receivers non-sources by construction.

The CLI is the only caller for now. It exposes stream-install and provides the
prototype unrar implementation with unrar lt for entry metadata and unrar p for
file bytes. This is simple and good enough to prove non-solid archive streaming,
but it is not the production provider shape for solid archives because per-file
unrar p would repeatedly decompress prefixes. The Tauri app explicitly passes
stream_install_provider: None, so the GUI behavior stays unchanged until a real
product path is designed.

Document the production-readiness work in NEXT_STEPS.md. The main follow-up is
to make the provider abstraction final-ish and replace the per-file CLI unrar
provider with a one-pass archive provider, then wire a deliberate GUI low-disk
mode, retry semantics, and broader failure scenarios.

Test Plan:
- just fmt
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just test
- python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py \
  S39 S40 --build-image
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just clippy
- git diff --check
- git diff --cached --check

Follow-up: NEXT_STEPS.md
This commit is contained in:
2026-06-07 20:31:51 +02:00
parent 8a8437036d
commit 373def6d44
22 changed files with 1465 additions and 14 deletions
+3
View File
@@ -14,11 +14,14 @@ path = "src/main.rs"
lanspread-compat = { path = "../lanspread-compat" }
lanspread-db = { path = "../lanspread-db" }
lanspread-peer = { path = "../lanspread-peer" }
lanspread-proto = { path = "../lanspread-proto" }
bytes = { workspace = true }
eyre = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true }
[lints.clippy]
needless_pass_by_value = "allow"
+4 -2
View File
@@ -4,14 +4,16 @@ WORKDIR /work
COPY . .
RUN cargo build --release -p lanspread-peer-cli
FROM debian:bookworm-slim
FROM debian:trixie-slim
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates \
&& apt-get install -y --no-install-recommends ca-certificates libstdc++6 \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /work/target/release/lanspread-peer-cli /usr/local/bin/lanspread-peer-cli
COPY crates/lanspread-tauri-deno-ts/src-tauri/game.db /app/game.db
COPY crates/lanspread-tauri-deno-ts/src-tauri/binaries/unrar-x86_64-unknown-linux-gnu /usr/local/bin/unrar
RUN chmod +x /usr/local/bin/unrar
ENTRYPOINT ["lanspread-peer-cli"]
CMD ["--games-dir", "/games", "--state-dir", "/state", "--catalog-db", "/app/game.db"]
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Run the peer-cli scenarios S1-S36 through Docker."""
"""Run the peer-cli scenarios S1-S40 through Docker."""
from __future__ import annotations
@@ -8,6 +8,7 @@ import hashlib
import json
import os
import queue
import shlex
import shutil
import subprocess
import sys
@@ -325,6 +326,8 @@ class Runner:
("S35", self.s35_unknown_game_filtered),
("S36", self.s36_latest_singleton),
("S37", self.s37_single_source_download_throughput),
("S39", self.s39_streamed_install_local_only),
("S40", self.s40_streamed_receiver_not_source),
]
for scenario_id, scenario in scenarios:
@@ -1060,6 +1063,114 @@ class Runner:
f"{throughput['chunks']} chunks"
)
def stream_install_cnctw(self, prefix: str) -> tuple[Peer, Peer]:
source_dir = self.fixture_root / f"{prefix}-bravo"
copy_game("cnctw", source_dir, version="20160128")
source = self.peer(f"{prefix}-bravo", games_dir=source_dir)
client = self.peer(f"{prefix}-client")
connect_many(client, [source])
wait_remote_game(client, "cnctw", peer_count=1)
waiter = LineWaiter(len(client.output))
client.send({"cmd": "stream-install", "game_id": "cnctw"})
client.wait_for(
event_is("got-game-files", "cnctw"),
timeout=20,
description="got cnctw files",
waiter=waiter,
)
client.wait_for(
event_is("download-begin", "cnctw"),
timeout=20,
description="stream begin cnctw",
waiter=waiter,
)
client.wait_for(
event_is("download-finished", "cnctw"),
timeout=60,
description="stream finish cnctw",
waiter=waiter,
)
client.wait_for(
event_is("install-finished", "cnctw"),
timeout=30,
description="stream install cnctw",
waiter=waiter,
)
return source, client
def s39_streamed_install_local_only(self) -> str:
source, client = self.stream_install_cnctw("s39")
game = wait_local_game(client, "cnctw", downloaded=False, installed=True)
assert_game_state(
game,
downloaded=False,
installed=True,
availability="LocalOnly",
)
game_root = client.host_games_dir / "cnctw"
assert_not_exists(game_root / "version.ini")
assert_not_exists(game_root / "cnctw.eti")
expected = {
"bin/cnctw-payload.bin": unrar_entry_sha256(
source, "cnctw", "bin/cnctw-payload.bin"
),
"data/cnctw-assets.dat": unrar_entry_sha256(
source, "cnctw", "data/cnctw-assets.dat"
),
}
actual = {
rel: sha256_file(game_root / "local" / rel)
for rel in expected
}
if actual != expected:
raise ScenarioError(f"streamed local payload hashes mismatched: {actual} != {expected}")
streamed_bytes = sum(
int(item.get("data", {}).get("length", 0))
for item in client.output
if item.get("type") == "event"
and item.get("event") == "download-chunk-finished"
and item.get("data", {}).get("game_id") == "cnctw"
)
expected_bytes = 3 * 1024 * 1024
if streamed_bytes != expected_bytes:
raise ScenarioError(
f"streamed byte count mismatch: {streamed_bytes} != {expected_bytes}"
)
return (
"cnctw streamed into local/ only; root archive and version.ini absent; "
f"payload hashes={actual}"
)
def s40_streamed_receiver_not_source(self) -> str:
_source, receiver = self.stream_install_cnctw("s40")
observer = self.peer("s40-observer")
connect_many(observer, [receiver])
receiver_snapshot = wait_peer_has_game(observer, receiver.peer_id, "cnctw")
summary = next(
game
for game in receiver_snapshot.get("games", [])
if game.get("id") == "cnctw"
)
if summary.get("availability") != "LocalOnly" or summary.get("downloaded"):
raise ScenarioError(f"receiver did not advertise cnctw as local-only: {summary}")
wait_remote_absent(observer, "cnctw", timeout=5)
err = observer.send(
{"cmd": "download", "game_id": "cnctw", "install": False},
expect_error=True,
)
if "no peers have game cnctw" not in err["error"]:
raise ScenarioError(f"unexpected local-only download error: {err}")
assert_not_exists(observer.host_games_dir / "cnctw")
return (
"observer saw receiver's local-only cnctw snapshot, but remote aggregation hid it "
f"and download errored '{err['error']}'"
)
def run(command: list[str], description: str) -> subprocess.CompletedProcess[str]:
result = subprocess.run(
@@ -1177,6 +1288,25 @@ def create_large_sparse_game(root: Path, *, size: int) -> None:
handle.truncate(size)
def sha256_file(path: Path) -> str:
hasher = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
hasher.update(chunk)
return hasher.hexdigest()
def unrar_entry_sha256(peer: Peer, game_id: str, relative_path: str) -> str:
command = (
f"unrar p -inul /games/{shlex.quote(game_id)}/{shlex.quote(game_id)}.eti "
f"{shlex.quote(relative_path)} | sha256sum"
)
output = peer.docker_exec("sh", "-c", command).stdout.strip()
if not output:
raise ScenarioError(f"empty sha256 output for {game_id}:{relative_path}")
return output.split()[0]
def format_bytes(size: int) -> str:
return f"{size / 1024 / 1024 / 1024:.2f} GiB"
+329 -1
View File
@@ -5,15 +5,21 @@
use std::{
net::SocketAddr,
path::{Path, PathBuf},
process::Stdio,
time::Duration,
};
use bytes::Bytes;
use eyre::{Context, OptionExt};
use lanspread_peer::{UnpackFuture, Unpacker};
use lanspread_peer::{StreamInstallFuture, StreamInstallProvider, UnpackFuture, Unpacker};
use lanspread_proto::StreamInstallFrame;
use serde::Serialize;
use serde_json::{Value, json};
use tokio::{io::AsyncReadExt, sync::mpsc};
use tokio_util::sync::CancellationToken;
pub const DEFAULT_FIXTURE_VERSION: &str = "20250101";
const STREAM_CHUNK_SIZE: usize = 256 * 1024;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CommandEnvelope {
@@ -33,6 +39,9 @@ pub enum CliCommand {
game_id: String,
install_after_download: bool,
},
StreamInstall {
game_id: String,
},
Install {
game_id: String,
},
@@ -63,6 +72,7 @@ impl CliCommand {
Self::ListGames => "list-games",
Self::SetGameDir { .. } => "set-game-dir",
Self::Download { .. } => "download",
Self::StreamInstall { .. } => "stream-install",
Self::Install { .. } => "install",
Self::Uninstall { .. } => "uninstall",
Self::Play { .. } => "play",
@@ -101,6 +111,9 @@ pub fn parse_command_value(value: &Value) -> eyre::Result<CommandEnvelope> {
game_id: game_id(object)?,
install_after_download: install_after_download(object)?,
},
"stream-install" => CliCommand::StreamInstall {
game_id: game_id(object)?,
},
"install" => CliCommand::Install {
game_id: game_id(object)?,
},
@@ -254,6 +267,270 @@ impl Unpacker for ExternalUnrarUnpacker {
}
}
pub struct ExternalUnrarStreamProvider {
program: PathBuf,
}
impl ExternalUnrarStreamProvider {
#[must_use]
pub fn new(program: PathBuf) -> Self {
Self { program }
}
}
impl StreamInstallProvider for ExternalUnrarStreamProvider {
fn stream_archive<'a>(
&'a self,
archive: &'a Path,
frames: mpsc::Sender<StreamInstallFrame>,
cancel_token: CancellationToken,
) -> StreamInstallFuture<'a> {
Box::pin(async move {
let listing = unrar_listing(&self.program, archive).await?;
let archive_name = archive
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("archive.eti")
.to_string();
send_stream_frame(
&frames,
StreamInstallFrame::ArchiveBegin {
archive_name: archive_name.clone(),
solid: listing.solid,
},
)
.await?;
for entry in listing.entries {
if cancel_token.is_cancelled() {
eyre::bail!("streamed archive {} was cancelled", archive.display());
}
match entry.kind {
RarEntryKind::Directory => {
send_stream_frame(
&frames,
StreamInstallFrame::Directory {
relative_path: entry.relative_path,
},
)
.await?;
}
RarEntryKind::File => {
send_stream_frame(
&frames,
StreamInstallFrame::FileBegin {
relative_path: entry.relative_path.clone(),
size: entry.size,
crc32: entry.crc32,
},
)
.await?;
stream_unrar_file(
&self.program,
archive,
&entry.relative_path,
&frames,
cancel_token.clone(),
)
.await?;
send_stream_frame(
&frames,
StreamInstallFrame::FileEnd {
relative_path: entry.relative_path,
},
)
.await?;
}
}
}
send_stream_frame(&frames, StreamInstallFrame::ArchiveEnd { archive_name }).await
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct RarListing {
solid: bool,
entries: Vec<RarEntry>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct RarEntry {
relative_path: String,
kind: RarEntryKind,
size: u64,
crc32: Option<u32>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RarEntryKind {
File,
Directory,
}
#[derive(Default)]
struct RarEntryDraft {
relative_path: Option<String>,
kind: Option<RarEntryKind>,
size: Option<u64>,
crc32: Option<u32>,
}
async fn unrar_listing(program: &Path, archive: &Path) -> eyre::Result<RarListing> {
let output = tokio::process::Command::new(program)
.arg("lt")
.arg("-cfg-")
.arg(archive)
.output()
.await?;
if !output.status.success() {
eyre::bail!(
"unrar lt failed for {} with status {}: {}",
archive.display(),
output.status,
String::from_utf8_lossy(&output.stderr)
);
}
parse_unrar_listing(&String::from_utf8_lossy(&output.stdout))
}
fn parse_unrar_listing(output: &str) -> eyre::Result<RarListing> {
let mut solid = false;
let mut entries = Vec::new();
let mut current = RarEntryDraft::default();
for line in output.lines() {
let trimmed = line.trim();
if let Some(details) = trimmed.strip_prefix("Details:") {
solid = details.to_ascii_lowercase().contains("solid");
continue;
}
if let Some(name) = trimmed.strip_prefix("Name:") {
push_rar_entry(&mut entries, std::mem::take(&mut current))?;
current.relative_path = Some(name.trim().to_string());
continue;
}
if let Some(kind) = trimmed.strip_prefix("Type:") {
current.kind = match kind.trim() {
"File" => Some(RarEntryKind::File),
"Directory" => Some(RarEntryKind::Directory),
_ => None,
};
continue;
}
if let Some(size) = trimmed.strip_prefix("Size:") {
current.size = Some(size.trim().parse()?);
continue;
}
if let Some(crc) = trimmed.strip_prefix("CRC32:") {
current.crc32 = Some(u32::from_str_radix(crc.trim(), 16)?);
}
}
push_rar_entry(&mut entries, current)?;
Ok(RarListing { solid, entries })
}
fn push_rar_entry(entries: &mut Vec<RarEntry>, draft: RarEntryDraft) -> eyre::Result<()> {
let Some(relative_path) = draft.relative_path else {
return Ok(());
};
let Some(kind) = draft.kind else {
return Ok(());
};
let size = match kind {
RarEntryKind::File => draft
.size
.ok_or_else(|| eyre::eyre!("RAR file entry {relative_path} has no Size"))?,
RarEntryKind::Directory => 0,
};
entries.push(RarEntry {
relative_path,
kind,
size,
crc32: draft.crc32,
});
Ok(())
}
async fn stream_unrar_file(
program: &Path,
archive: &Path,
relative_path: &str,
frames: &mpsc::Sender<StreamInstallFrame>,
cancel_token: CancellationToken,
) -> eyre::Result<()> {
let mut child = tokio::process::Command::new(program)
.arg("p")
.arg("-inul")
.arg("-cfg-")
.arg(archive)
.arg(relative_path)
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()?;
let mut stdout = child
.stdout
.take()
.ok_or_eyre("unrar stdout was not captured")?;
let mut buffer = vec![0_u8; STREAM_CHUNK_SIZE];
loop {
let read = tokio::select! {
() = cancel_token.cancelled() => {
let _ = child.kill().await;
eyre::bail!("streaming {relative_path} from {} was cancelled", archive.display());
}
read = stdout.read(&mut buffer) => read?,
};
if read == 0 {
break;
}
send_stream_frame(
frames,
StreamInstallFrame::FileChunk {
bytes: Bytes::copy_from_slice(&buffer[..read]),
},
)
.await?;
}
let status = child.wait().await?;
if !status.success() {
eyre::bail!(
"unrar p failed for {}:{} with status {status}",
archive.display(),
relative_path
);
}
Ok(())
}
async fn send_stream_frame(
frames: &mpsc::Sender<StreamInstallFrame>,
frame: StreamInstallFrame,
) -> eyre::Result<()> {
frames
.send(frame)
.await
.map_err(|_| eyre::eyre!("streamed install frame receiver closed"))
}
pub fn result_line(id: &Option<Value>, command: &str, data: Value) -> eyre::Result<String> {
output_line(json!({
"type": "result",
@@ -344,6 +621,57 @@ mod tests {
assert_eq!(parsed["data"]["peer_count"], 0);
}
#[test]
fn parses_stream_install_command() {
let parsed = parse_command_line(r#"{"cmd":"stream-install","game_id":"cnctw"}"#)
.expect("command should parse");
assert_eq!(
parsed.command,
CliCommand::StreamInstall {
game_id: "cnctw".to_string(),
}
);
}
#[test]
fn parses_unrar_technical_listing() {
let listing = parse_unrar_listing(
r#"
Archive: game.eti
Details: RAR 5
Name: bin/payload.bin
Type: File
Size: 123
CRC32: 38B488A7
Name: bin
Type: Directory
"#,
)
.expect("listing should parse");
assert!(!listing.solid);
assert_eq!(
listing.entries,
vec![
RarEntry {
relative_path: "bin/payload.bin".to_string(),
kind: RarEntryKind::File,
size: 123,
crc32: Some(0x38B4_88A7),
},
RarEntry {
relative_path: "bin".to_string(),
kind: RarEntryKind::Directory,
size: 0,
crc32: None,
},
]
);
}
#[tokio::test]
async fn fixture_unpacker_creates_install_payload() {
let temp = TempDir::new("lanspread-peer-cli-fixture");
+28 -1
View File
@@ -17,6 +17,7 @@ use lanspread_peer::{
ActiveOperation,
ActiveOperationKind,
InstallOperation,
NoopStreamInstallProvider,
PeerCommand,
PeerEvent,
PeerGameDB,
@@ -24,6 +25,7 @@ use lanspread_peer::{
PeerRuntimeHandle,
PeerSnapshot,
PeerStartOptions,
StreamInstallProvider,
migrate_legacy_state,
start_peer_with_options,
};
@@ -31,6 +33,7 @@ use lanspread_peer_cli::{
CliCommand,
CommandEnvelope,
DEFAULT_FIXTURE_VERSION,
ExternalUnrarStreamProvider,
ExternalUnrarUnpacker,
FixtureSeed,
FixtureUnpacker,
@@ -134,10 +137,15 @@ async fn main() -> eyre::Result<()> {
let (tx_events, rx_events) = mpsc::unbounded_channel();
let peer_game_db = Arc::new(RwLock::new(PeerGameDB::new()));
let catalog = Arc::new(RwLock::new(catalog));
let unpacker: Arc<dyn lanspread_peer::Unpacker> = match args.unrar {
let unrar_for_streaming = args.unrar.clone().or_else(default_unrar_program);
let unpacker: Arc<dyn lanspread_peer::Unpacker> = match args.unrar.clone() {
Some(path) => Arc::new(ExternalUnrarUnpacker::new(path)),
None => Arc::new(FixtureUnpacker),
};
let stream_install_provider: Arc<dyn StreamInstallProvider> = match unrar_for_streaming {
Some(path) => Arc::new(ExternalUnrarStreamProvider::new(path)),
None => Arc::new(NoopStreamInstallProvider),
};
let mut handle = start_peer_with_options(
args.games_dir.clone(),
@@ -148,6 +156,7 @@ async fn main() -> eyre::Result<()> {
PeerStartOptions {
state_dir: Some(args.state_dir.clone()),
active_outbound_transfers: None,
stream_install_provider: Some(stream_install_provider),
},
)?;
let sender = handle.sender();
@@ -249,6 +258,15 @@ async fn handle_command(
})?;
Ok(json!({"queued": true, "game_id": game_id, "install": install_after_download}))
}
CliCommand::StreamInstall { game_id } => {
ensure_catalog_game(shared, game_id).await?;
ensure_no_active_operation(shared, game_id).await?;
let _ = game_files_for_download(sender, shared, game_id).await?;
sender.send(PeerCommand::StreamInstallGame {
id: game_id.clone(),
})?;
Ok(json!({"queued": true, "game_id": game_id}))
}
CliCommand::Install { game_id } => {
ensure_catalog_game(shared, game_id).await?;
ensure_no_active_operation(shared, game_id).await?;
@@ -729,6 +747,15 @@ fn default_catalog_db() -> Option<PathBuf> {
.find(|path| path.exists())
}
fn default_unrar_program() -> Option<PathBuf> {
[
PathBuf::from("/usr/local/bin/unrar"),
PathBuf::from("/usr/bin/unrar"),
]
.into_iter()
.find(|path| path.exists())
}
fn next_string(args: &mut impl Iterator<Item = OsString>, flag: &str) -> eyre::Result<String> {
args.next()
.ok_or_else(|| eyre::eyre!("{flag} requires a value"))?