ddidderr/lanspread
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(stream-install)!: stream archive payloads as raw frames

Browse Source 
Streamed installs were sending FileChunk payloads through the shared JSON
Message impl. serde_json serializes bytes as arrays of integers, which
bloats wire traffic and burns CPU on large archives. Replace
StreamInstallFrame encoding with tagged frames: JSON control frames keep
their shape under tag 0, while file chunks carry raw bytes under tag 1.

The stream install metadata now carries unpacked archive size and mandatory
CRC32. The CLI unrar provider validates CRCs up front, runs one
archive-wide unrar p stream, splits stdout by listed file sizes, and
refuses trailing or missing bytes. That avoids solid archive
re-decompression and sidesteps unrar wildcard masks for path arguments.

Receivers now sample existing download progress events for streamed
installs, report staging-relative chunk paths, and retry trusted peers with
a fresh streamed-install transaction after a failed attempt. The current
protocol policy does not preserve compatibility with older stream-install
builds.

Test Plan:
- just fmt
- just test
- just clippy
- git diff --check
- git diff --cached --check

BREAKING CHANGE: StreamInstallFrame now uses tagged frames with raw chunk
payloads and requires current peers on both sides of streamed installs.

Refs: NEXT_STEPS_CLAUDES_REVIEW.md
This commit is contained in:
ddidderr
2026-06-07 21:12:15 +02:00
parent cc147def73
commit 5dd356eca8
5 changed files with 450 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/lanspread-peer/src/stream_install.rs
+90 -15
View File
@@ -4,6 +4,7 @@ use std::{
path::{Path, PathBuf},
pin::Pin,
sync::Arc,
time::{Duration, Instant},
};
use bytes::Bytes;
@@ -15,6 +16,7 @@ use tokio::{
fs::File,
io::AsyncWriteExt,
sync::{mpsc, mpsc::UnboundedSender},
time::{self, MissedTickBehavior},
};
use tokio_util::{
codec::{FramedRead, FramedWrite, LengthDelimitedCodec},
@@ -22,6 +24,7 @@ use tokio_util::{
};
use crate::{
DownloadProgress,
PeerEvent,
install::root_eti_archives,
network::connect_to_peer,
@@ -29,6 +32,7 @@ use crate::{
};
const FRAME_CHANNEL_DEPTH: usize = 16;
const STREAM_INSTALL_PROGRESS_UPDATE_INTERVAL: Duration = Duration::from_millis(500);
pub type StreamInstallFuture<'a> = Pin<Box<dyn Future<Output = eyre::Result<()>> + Send + 'a>>;
@@ -182,10 +186,18 @@ pub(crate) async fn receive_streamed_install(
let mut framed_rx = FramedRead::new(rx, LengthDelimitedCodec::new());
let mut current_file: Option<IncomingFile> = None;
let mut progress = StreamInstallProgress::new(game_id.to_string());
let mut progress_interval = time::interval(STREAM_INSTALL_PROGRESS_UPDATE_INTERVAL);
progress_interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
progress_interval.tick().await;
loop {
let next = tokio::select! {
() = cancel_token.cancelled() => eyre::bail!("streamed install for {game_id} was cancelled"),
_ = progress_interval.tick() => {
progress.emit_current(&tx_notify_ui);
continue;
}
next = framed_rx.next() => next,
};
@@ -199,9 +211,13 @@ pub(crate) async fn receive_streamed_install(
StreamInstallFrame::ArchiveBegin {
archive_name,
solid,
unpacked_size,
} => {
progress.add_total(unpacked_size);
progress.emit_snapshot(&tx_notify_ui, 0);
log::info!(
"Receiving streamed install archive {archive_name} for {game_id} (solid={solid})"
"Receiving streamed install archive {archive_name} for {game_id} \
(solid={solid}, unpacked_size={unpacked_size})"
);
}
StreamInstallFrame::Directory { relative_path } => {
@@ -227,8 +243,10 @@ pub(crate) async fn receive_streamed_install(
let Some(file) = current_file.as_mut() else {
eyre::bail!("received FileChunk without FileBegin");
};
file.write_chunk(game_id, peer_addr, &tx_notify_ui, bytes)
let length = file
.write_chunk(game_id, peer_addr, &tx_notify_ui, bytes)
.await?;
progress.record_bytes(length);
}
StreamInstallFrame::FileEnd { relative_path } => {
let Some(file) = current_file.take() else {
@@ -243,6 +261,7 @@ pub(crate) async fn receive_streamed_install(
if current_file.is_some() {
eyre::bail!("streamed install completed with an open file");
}
progress.emit_snapshot(&tx_notify_ui, 0);
return Ok(());
}
StreamInstallFrame::Error { message } => {
@@ -252,11 +271,68 @@ pub(crate) async fn receive_streamed_install(
}
}
struct StreamInstallProgress {
id: String,
total_bytes: u64,
downloaded_bytes: u64,
last_downloaded_bytes: u64,
last_at: Instant,
}
impl StreamInstallProgress {
fn new(id: String) -> Self {
Self {
id,
total_bytes: 0,
downloaded_bytes: 0,
last_downloaded_bytes: 0,
last_at: Instant::now(),
}
}
fn add_total(&mut self, bytes: u64) {
self.total_bytes = self.total_bytes.saturating_add(bytes);
}
fn record_bytes(&mut self, bytes: u64) {
self.downloaded_bytes = self.downloaded_bytes.saturating_add(bytes);
}
fn emit_current(&mut self, tx_notify_ui: &UnboundedSender<PeerEvent>) {
let now = Instant::now();
let speed = bytes_per_second(
self.downloaded_bytes
.saturating_sub(self.last_downloaded_bytes),
now.duration_since(self.last_at),
);
self.last_downloaded_bytes = self.downloaded_bytes;
self.last_at = now;
self.emit_snapshot(tx_notify_ui, speed);
}
fn emit_snapshot(&self, tx_notify_ui: &UnboundedSender<PeerEvent>, bytes_per_second: u64) {
let _ = tx_notify_ui.send(PeerEvent::DownloadGameFilesProgress(DownloadProgress {
id: self.id.clone(),
downloaded_bytes: self.downloaded_bytes,
total_bytes: self.total_bytes,
bytes_per_second,
active_peer_count: 1,
}));
}
}
fn bytes_per_second(bytes: u64, elapsed: Duration) -> u64 {
let millis = elapsed.as_millis().max(1);
let rate = u128::from(bytes).saturating_mul(1_000) / millis;
u64::try_from(rate).unwrap_or(u64::MAX)
}
struct IncomingFile {
relative_path: String,
path: PathBuf,
expected_size: u64,
expected_crc32: Option<u32>,
expected_crc32: u32,
received: u64,
hasher: Hasher,
file: File,
@@ -267,7 +343,7 @@ impl IncomingFile {
relative_path: String,
path: PathBuf,
expected_size: u64,
expected_crc32: Option<u32>,
expected_crc32: u32,
file: File,
) -> Self {
Self {
@@ -287,7 +363,7 @@ impl IncomingFile {
peer_addr: SocketAddr,
tx_notify_ui: &UnboundedSender<PeerEvent>,
bytes: Bytes,
) -> eyre::Result<()> {
) -> eyre::Result<u64> {
let offset = self.received;
let length = u64::try_from(bytes.len())?;
if offset.saturating_add(length) > self.expected_size {
@@ -304,11 +380,11 @@ impl IncomingFile {
let _ = tx_notify_ui.send(PeerEvent::DownloadGameFileChunkFinished {
id: game_id.to_string(),
peer_addr,
relative_path: format!("{game_id}/local/{}", self.relative_path),
relative_path: format!("{game_id}/.local.installing/{}", self.relative_path),
offset,
length,
});
Ok(())
Ok(length)
}
async fn finish(mut self, relative_path: &str) -> eyre::Result<()> {
@@ -329,14 +405,13 @@ impl IncomingFile {
);
}
if let Some(expected) = self.expected_crc32 {
let actual = self.hasher.finalize();
if actual != expected {
eyre::bail!(
"streamed file {} CRC32 mismatch: got {actual:08X}, expected {expected:08X}",
self.relative_path
);
}
let actual = self.hasher.finalize();
if actual != self.expected_crc32 {
eyre::bail!(
"streamed file {} CRC32 mismatch: got {actual:08X}, expected {:08X}",
self.relative_path,
self.expected_crc32
);
}
log::debug!(