feat(peer): prototype streamed installs

Add a streamed-install prototype that can receive archive-derived install bytes straight into local/ without first storing the peer-owned root archive payload. This is intended for low-disk clients that want to install a game but opt out of becoming a downloadable peer source for that game. The protocol gains a current-version-only StreamInstall request and framed StreamInstallFrame responses. The peer core owns the generic transport, transaction, path validation, size checks, CRC32 verification, and lifecycle state. The archive-specific work is hidden behind StreamInstallProvider so the prototype can use unrar while the final implementation can swap in a better provider without rewriting the peer command path. The receiver writes into .local.installing and only promotes to local/ after the full stream verifies. It deliberately does not write the root version.ini or archive files, so the settled local state is installed=true, downloaded=false, and availability=LocalOnly. That preserves the existing rule that local/ is not served to peers and makes streamed receivers non-sources by construction. The CLI is the only caller for now. It exposes stream-install and provides the prototype unrar implementation with unrar lt for entry metadata and unrar p for file bytes. This is simple and good enough to prove non-solid archive streaming, but it is not the production provider shape for solid archives because per-file unrar p would repeatedly decompress prefixes. The Tauri app explicitly passes stream_install_provider: None, so the GUI behavior stays unchanged until a real product path is designed. Document the production-readiness work in NEXT_STEPS.md. The main follow-up is to make the provider abstraction final-ish and replace the per-file CLI unrar provider with a one-pass archive provider, then wire a deliberate GUI low-disk mode, retry semantics, and broader failure scenarios. Test Plan: - just fmt - RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just test - python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py \ S39 S40 --build-image - RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just clippy - git diff --check - git diff --cached --check Follow-up: NEXT_STEPS.md
2026-06-07 20:31:51 +02:00
parent 8a8437036d
commit 373def6d44
22 changed files with 1465 additions and 14 deletions
@@ -0,0 +1,372 @@
+use std::{
+    future::Future,
+    net::SocketAddr,
+    path::{Path, PathBuf},
+    pin::Pin,
+    sync::Arc,
+};
+
+use bytes::Bytes;
+use crc32fast::Hasher;
+use futures::{SinkExt, StreamExt};
+use lanspread_proto::{Message, Request, StreamInstallFrame};
+use s2n_quic::stream::SendStream;
+use tokio::{
+    fs::File,
+    io::AsyncWriteExt,
+    sync::{mpsc, mpsc::UnboundedSender},
+};
+use tokio_util::{
+    codec::{FramedRead, FramedWrite, LengthDelimitedCodec},
+    sync::CancellationToken,
+};
+
+use crate::{
+    PeerEvent,
+    install::root_eti_archives,
+    network::connect_to_peer,
+    path_validation::validate_game_file_path,
+};
+
+const FRAME_CHANNEL_DEPTH: usize = 16;
+
+pub type StreamInstallFuture<'a> = Pin<Box<dyn Future<Output = eyre::Result<()>> + Send + 'a>>;
+
+pub trait StreamInstallProvider: Send + Sync {
+    fn stream_archive<'a>(
+        &'a self,
+        archive: &'a Path,
+        frames: mpsc::Sender<StreamInstallFrame>,
+        cancel_token: CancellationToken,
+    ) -> StreamInstallFuture<'a>;
+}
+
+#[derive(Debug, Default)]
+pub struct NoopStreamInstallProvider;
+
+impl StreamInstallProvider for NoopStreamInstallProvider {
+    fn stream_archive<'a>(
+        &'a self,
+        archive: &'a Path,
+        _frames: mpsc::Sender<StreamInstallFrame>,
+        _cancel_token: CancellationToken,
+    ) -> StreamInstallFuture<'a> {
+        Box::pin(async move {
+            eyre::bail!(
+                "streamed install provider is not configured for {}",
+                archive.display()
+            )
+        })
+    }
+}
+
+pub(crate) async fn send_stream_install_error(
+    tx: SendStream,
+    message: impl Into<String>,
+) -> SendStream {
+    let mut framed_tx = FramedWrite::new(tx, LengthDelimitedCodec::new());
+    if let Err(err) = framed_tx
+        .send(
+            StreamInstallFrame::Error {
+                message: message.into(),
+            }
+            .encode(),
+        )
+        .await
+    {
+        log::warn!("Failed to send streamed install error frame: {err}");
+    }
+    if let Err(err) = framed_tx.close().await {
+        log::debug!("Failed to close streamed install error response: {err}");
+    }
+    framed_tx.into_inner()
+}
+
+pub(crate) async fn send_game_install_stream(
+    provider: Arc<dyn StreamInstallProvider>,
+    tx: SendStream,
+    game_root: &Path,
+    game_id: &str,
+    cancel_token: CancellationToken,
+) -> (SendStream, eyre::Result<()>) {
+    let archives = match root_eti_archives(game_root).await {
+        Ok(archives) => archives,
+        Err(err) => {
+            let message = err.to_string();
+            let tx = send_stream_install_error(tx, message.clone()).await;
+            return (tx, Err(eyre::eyre!(message)));
+        }
+    };
+    if archives.is_empty() {
+        let message = format!("no .eti archives found for {game_id}");
+        let tx = send_stream_install_error(tx, message.clone()).await;
+        return (tx, Err(eyre::eyre!(message)));
+    }
+
+    let (frame_tx, mut frame_rx) = mpsc::channel(FRAME_CHANNEL_DEPTH);
+    let producer_cancel = cancel_token.child_token();
+    let game_id_for_producer = game_id.to_string();
+    let producer = tokio::spawn({
+        let provider = provider.clone();
+        let producer_cancel = producer_cancel.clone();
+        async move {
+            for archive in archives {
+                if producer_cancel.is_cancelled() {
+                    eyre::bail!("streamed install for {game_id_for_producer} was cancelled");
+                }
+
+                if let Err(err) = provider
+                    .stream_archive(&archive, frame_tx.clone(), producer_cancel.clone())
+                    .await
+                {
+                    let message = err.to_string();
+                    let _ = frame_tx.send(StreamInstallFrame::Error { message }).await;
+                    return Err(err);
+                }
+            }
+
+            let _ = frame_tx.send(StreamInstallFrame::Complete).await;
+            Ok(())
+        }
+    });
+
+    let mut framed_tx = FramedWrite::new(tx, LengthDelimitedCodec::new());
+    let mut send_result = Ok(());
+
+    while let Some(frame) = frame_rx.recv().await {
+        if let Err(err) = framed_tx.send(frame.encode()).await {
+            producer_cancel.cancel();
+            send_result = Err(eyre::eyre!("failed to send streamed install frame: {err}"));
+            break;
+        }
+    }
+
+    let close_result = framed_tx
+        .close()
+        .await
+        .map_err(|err| eyre::eyre!("failed to close streamed install stream: {err}"));
+    let tx = framed_tx.into_inner();
+    let producer_result = match producer.await {
+        Ok(result) => result,
+        Err(err) => Err(eyre::eyre!("streamed install producer task failed: {err}")),
+    };
+    let result = send_result.and(producer_result).and(close_result);
+
+    (tx, result)
+}
+
+pub(crate) async fn receive_streamed_install(
+    peer_addr: SocketAddr,
+    game_id: &str,
+    staging_dir: &Path,
+    tx_notify_ui: UnboundedSender<PeerEvent>,
+    cancel_token: CancellationToken,
+) -> eyre::Result<()> {
+    let staging_dir = tokio::fs::canonicalize(staging_dir)
+        .await
+        .unwrap_or_else(|_| staging_dir.to_path_buf());
+    let mut conn = connect_to_peer(peer_addr).await?;
+    let stream = conn.open_bidirectional_stream().await?;
+    let (rx, tx) = stream.split();
+    let mut framed_tx = FramedWrite::new(tx, LengthDelimitedCodec::new());
+
+    framed_tx
+        .send(
+            Request::StreamInstall {
+                game_id: game_id.to_string(),
+            }
+            .encode(),
+        )
+        .await?;
+    framed_tx.close().await?;
+
+    let mut framed_rx = FramedRead::new(rx, LengthDelimitedCodec::new());
+    let mut current_file: Option<IncomingFile> = None;
+
+    loop {
+        let next = tokio::select! {
+            () = cancel_token.cancelled() => eyre::bail!("streamed install for {game_id} was cancelled"),
+            next = framed_rx.next() => next,
+        };
+
+        let Some(frame) = next else {
+            eyre::bail!("streamed install ended before Complete");
+        };
+        let frame = frame?.freeze();
+        let frame = StreamInstallFrame::decode(frame);
+
+        match frame {
+            StreamInstallFrame::ArchiveBegin {
+                archive_name,
+                solid,
+            } => {
+                log::info!(
+                    "Receiving streamed install archive {archive_name} for {game_id} (solid={solid})"
+                );
+            }
+            StreamInstallFrame::Directory { relative_path } => {
+                let path = resolve_stream_path(&staging_dir, &relative_path)?;
+                tokio::fs::create_dir_all(path).await?;
+            }
+            StreamInstallFrame::FileBegin {
+                relative_path,
+                size,
+                crc32,
+            } => {
+                if current_file.is_some() {
+                    eyre::bail!("received FileBegin for {relative_path} before previous FileEnd");
+                }
+                let path = resolve_stream_path(&staging_dir, &relative_path)?;
+                if let Some(parent) = path.parent() {
+                    tokio::fs::create_dir_all(parent).await?;
+                }
+                let file = File::create(&path).await?;
+                current_file = Some(IncomingFile::new(relative_path, path, size, crc32, file));
+            }
+            StreamInstallFrame::FileChunk { bytes } => {
+                let Some(file) = current_file.as_mut() else {
+                    eyre::bail!("received FileChunk without FileBegin");
+                };
+                file.write_chunk(game_id, peer_addr, &tx_notify_ui, bytes)
+                    .await?;
+            }
+            StreamInstallFrame::FileEnd { relative_path } => {
+                let Some(file) = current_file.take() else {
+                    eyre::bail!("received FileEnd for {relative_path} without FileBegin");
+                };
+                file.finish(&relative_path).await?;
+            }
+            StreamInstallFrame::ArchiveEnd { archive_name } => {
+                log::info!("Finished streamed install archive {archive_name} for {game_id}");
+            }
+            StreamInstallFrame::Complete => {
+                if current_file.is_some() {
+                    eyre::bail!("streamed install completed with an open file");
+                }
+                return Ok(());
+            }
+            StreamInstallFrame::Error { message } => {
+                eyre::bail!("streamed install sender failed: {message}");
+            }
+        }
+    }
+}
+
+struct IncomingFile {
+    relative_path: String,
+    path: PathBuf,
+    expected_size: u64,
+    expected_crc32: Option<u32>,
+    received: u64,
+    hasher: Hasher,
+    file: File,
+}
+
+impl IncomingFile {
+    fn new(
+        relative_path: String,
+        path: PathBuf,
+        expected_size: u64,
+        expected_crc32: Option<u32>,
+        file: File,
+    ) -> Self {
+        Self {
+            relative_path,
+            path,
+            expected_size,
+            expected_crc32,
+            received: 0,
+            hasher: Hasher::new(),
+            file,
+        }
+    }
+
+    async fn write_chunk(
+        &mut self,
+        game_id: &str,
+        peer_addr: SocketAddr,
+        tx_notify_ui: &UnboundedSender<PeerEvent>,
+        bytes: Bytes,
+    ) -> eyre::Result<()> {
+        let offset = self.received;
+        let length = u64::try_from(bytes.len())?;
+        if offset.saturating_add(length) > self.expected_size {
+            eyre::bail!(
+                "streamed file {} exceeded expected size {}",
+                self.relative_path,
+                self.expected_size
+            );
+        }
+        self.file.write_all(&bytes).await?;
+        self.hasher.update(&bytes);
+        self.received = self.received.saturating_add(length);
+
+        let _ = tx_notify_ui.send(PeerEvent::DownloadGameFileChunkFinished {
+            id: game_id.to_string(),
+            peer_addr,
+            relative_path: format!("{game_id}/local/{}", self.relative_path),
+            offset,
+            length,
+        });
+        Ok(())
+    }
+
+    async fn finish(mut self, relative_path: &str) -> eyre::Result<()> {
+        if self.relative_path != relative_path {
+            eyre::bail!(
+                "streamed file end mismatch: began {}, ended {relative_path}",
+                self.relative_path
+            );
+        }
+        self.file.flush().await?;
+
+        if self.received != self.expected_size {
+            eyre::bail!(
+                "streamed file {} size mismatch: got {}, expected {}",
+                self.relative_path,
+                self.received,
+                self.expected_size
+            );
+        }
+
+        if let Some(expected) = self.expected_crc32 {
+            let actual = self.hasher.finalize();
+            if actual != expected {
+                eyre::bail!(
+                    "streamed file {} CRC32 mismatch: got {actual:08X}, expected {expected:08X}",
+                    self.relative_path
+                );
+            }
+        }
+
+        log::debug!(
+            "Received streamed file {} -> {}",
+            self.relative_path,
+            self.path.display()
+        );
+        Ok(())
+    }
+}
+
+fn resolve_stream_path(staging_dir: &Path, relative_path: &str) -> eyre::Result<PathBuf> {
+    validate_game_file_path(staging_dir, relative_path)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_support::TempDir;
+
+    #[test]
+    fn stream_paths_stay_inside_staging_dir() {
+        let temp = TempDir::new("lanspread-stream-install-path");
+        let staging = temp.path().join("staging");
+        std::fs::create_dir_all(&staging).expect("staging should be created");
+        let staging = std::fs::canonicalize(staging).expect("staging should canonicalize");
+
+        assert!(resolve_stream_path(&staging, "bin/game.exe").is_ok());
+        assert!(resolve_stream_path(&staging, "../outside").is_err());
+        assert!(resolve_stream_path(&staging, "/absolute").is_err());
+        assert!(resolve_stream_path(&staging, "C:/windows").is_err());
+    }
+}