feat(peer): prototype streamed installs
Add a streamed-install prototype that can receive archive-derived install bytes straight into local/ without first storing the peer-owned root archive payload. This is intended for low-disk clients that want to install a game but opt out of becoming a downloadable peer source for that game. The protocol gains a current-version-only StreamInstall request and framed StreamInstallFrame responses. The peer core owns the generic transport, transaction, path validation, size checks, CRC32 verification, and lifecycle state. The archive-specific work is hidden behind StreamInstallProvider so the prototype can use unrar while the final implementation can swap in a better provider without rewriting the peer command path. The receiver writes into .local.installing and only promotes to local/ after the full stream verifies. It deliberately does not write the root version.ini or archive files, so the settled local state is installed=true, downloaded=false, and availability=LocalOnly. That preserves the existing rule that local/ is not served to peers and makes streamed receivers non-sources by construction. The CLI is the only caller for now. It exposes stream-install and provides the prototype unrar implementation with unrar lt for entry metadata and unrar p for file bytes. This is simple and good enough to prove non-solid archive streaming, but it is not the production provider shape for solid archives because per-file unrar p would repeatedly decompress prefixes. The Tauri app explicitly passes stream_install_provider: None, so the GUI behavior stays unchanged until a real product path is designed. Document the production-readiness work in NEXT_STEPS.md. The main follow-up is to make the provider abstraction final-ish and replace the per-file CLI unrar provider with a one-pass archive provider, then wire a deliberate GUI low-disk mode, retry semantics, and broader failure scenarios. Test Plan: - just fmt - RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just test - python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py \ S39 S40 --build-image - RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just clippy - git diff --check - git diff --cached --check Follow-up: NEXT_STEPS.md
This commit is contained in:
@@ -5,15 +5,21 @@
|
||||
use std::{
|
||||
net::SocketAddr,
|
||||
path::{Path, PathBuf},
|
||||
process::Stdio,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use bytes::Bytes;
|
||||
use eyre::{Context, OptionExt};
|
||||
use lanspread_peer::{UnpackFuture, Unpacker};
|
||||
use lanspread_peer::{StreamInstallFuture, StreamInstallProvider, UnpackFuture, Unpacker};
|
||||
use lanspread_proto::StreamInstallFrame;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::{io::AsyncReadExt, sync::mpsc};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub const DEFAULT_FIXTURE_VERSION: &str = "20250101";
|
||||
const STREAM_CHUNK_SIZE: usize = 256 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CommandEnvelope {
|
||||
@@ -33,6 +39,9 @@ pub enum CliCommand {
|
||||
game_id: String,
|
||||
install_after_download: bool,
|
||||
},
|
||||
StreamInstall {
|
||||
game_id: String,
|
||||
},
|
||||
Install {
|
||||
game_id: String,
|
||||
},
|
||||
@@ -63,6 +72,7 @@ impl CliCommand {
|
||||
Self::ListGames => "list-games",
|
||||
Self::SetGameDir { .. } => "set-game-dir",
|
||||
Self::Download { .. } => "download",
|
||||
Self::StreamInstall { .. } => "stream-install",
|
||||
Self::Install { .. } => "install",
|
||||
Self::Uninstall { .. } => "uninstall",
|
||||
Self::Play { .. } => "play",
|
||||
@@ -101,6 +111,9 @@ pub fn parse_command_value(value: &Value) -> eyre::Result<CommandEnvelope> {
|
||||
game_id: game_id(object)?,
|
||||
install_after_download: install_after_download(object)?,
|
||||
},
|
||||
"stream-install" => CliCommand::StreamInstall {
|
||||
game_id: game_id(object)?,
|
||||
},
|
||||
"install" => CliCommand::Install {
|
||||
game_id: game_id(object)?,
|
||||
},
|
||||
@@ -254,6 +267,270 @@ impl Unpacker for ExternalUnrarUnpacker {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ExternalUnrarStreamProvider {
|
||||
program: PathBuf,
|
||||
}
|
||||
|
||||
impl ExternalUnrarStreamProvider {
|
||||
#[must_use]
|
||||
pub fn new(program: PathBuf) -> Self {
|
||||
Self { program }
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamInstallProvider for ExternalUnrarStreamProvider {
|
||||
fn stream_archive<'a>(
|
||||
&'a self,
|
||||
archive: &'a Path,
|
||||
frames: mpsc::Sender<StreamInstallFrame>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> StreamInstallFuture<'a> {
|
||||
Box::pin(async move {
|
||||
let listing = unrar_listing(&self.program, archive).await?;
|
||||
let archive_name = archive
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("archive.eti")
|
||||
.to_string();
|
||||
|
||||
send_stream_frame(
|
||||
&frames,
|
||||
StreamInstallFrame::ArchiveBegin {
|
||||
archive_name: archive_name.clone(),
|
||||
solid: listing.solid,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
for entry in listing.entries {
|
||||
if cancel_token.is_cancelled() {
|
||||
eyre::bail!("streamed archive {} was cancelled", archive.display());
|
||||
}
|
||||
|
||||
match entry.kind {
|
||||
RarEntryKind::Directory => {
|
||||
send_stream_frame(
|
||||
&frames,
|
||||
StreamInstallFrame::Directory {
|
||||
relative_path: entry.relative_path,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
RarEntryKind::File => {
|
||||
send_stream_frame(
|
||||
&frames,
|
||||
StreamInstallFrame::FileBegin {
|
||||
relative_path: entry.relative_path.clone(),
|
||||
size: entry.size,
|
||||
crc32: entry.crc32,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
stream_unrar_file(
|
||||
&self.program,
|
||||
archive,
|
||||
&entry.relative_path,
|
||||
&frames,
|
||||
cancel_token.clone(),
|
||||
)
|
||||
.await?;
|
||||
send_stream_frame(
|
||||
&frames,
|
||||
StreamInstallFrame::FileEnd {
|
||||
relative_path: entry.relative_path,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
send_stream_frame(&frames, StreamInstallFrame::ArchiveEnd { archive_name }).await
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct RarListing {
|
||||
solid: bool,
|
||||
entries: Vec<RarEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct RarEntry {
|
||||
relative_path: String,
|
||||
kind: RarEntryKind,
|
||||
size: u64,
|
||||
crc32: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum RarEntryKind {
|
||||
File,
|
||||
Directory,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RarEntryDraft {
|
||||
relative_path: Option<String>,
|
||||
kind: Option<RarEntryKind>,
|
||||
size: Option<u64>,
|
||||
crc32: Option<u32>,
|
||||
}
|
||||
|
||||
async fn unrar_listing(program: &Path, archive: &Path) -> eyre::Result<RarListing> {
|
||||
let output = tokio::process::Command::new(program)
|
||||
.arg("lt")
|
||||
.arg("-cfg-")
|
||||
.arg(archive)
|
||||
.output()
|
||||
.await?;
|
||||
if !output.status.success() {
|
||||
eyre::bail!(
|
||||
"unrar lt failed for {} with status {}: {}",
|
||||
archive.display(),
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
parse_unrar_listing(&String::from_utf8_lossy(&output.stdout))
|
||||
}
|
||||
|
||||
fn parse_unrar_listing(output: &str) -> eyre::Result<RarListing> {
|
||||
let mut solid = false;
|
||||
let mut entries = Vec::new();
|
||||
let mut current = RarEntryDraft::default();
|
||||
|
||||
for line in output.lines() {
|
||||
let trimmed = line.trim();
|
||||
if let Some(details) = trimmed.strip_prefix("Details:") {
|
||||
solid = details.to_ascii_lowercase().contains("solid");
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(name) = trimmed.strip_prefix("Name:") {
|
||||
push_rar_entry(&mut entries, std::mem::take(&mut current))?;
|
||||
current.relative_path = Some(name.trim().to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(kind) = trimmed.strip_prefix("Type:") {
|
||||
current.kind = match kind.trim() {
|
||||
"File" => Some(RarEntryKind::File),
|
||||
"Directory" => Some(RarEntryKind::Directory),
|
||||
_ => None,
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(size) = trimmed.strip_prefix("Size:") {
|
||||
current.size = Some(size.trim().parse()?);
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(crc) = trimmed.strip_prefix("CRC32:") {
|
||||
current.crc32 = Some(u32::from_str_radix(crc.trim(), 16)?);
|
||||
}
|
||||
}
|
||||
|
||||
push_rar_entry(&mut entries, current)?;
|
||||
Ok(RarListing { solid, entries })
|
||||
}
|
||||
|
||||
fn push_rar_entry(entries: &mut Vec<RarEntry>, draft: RarEntryDraft) -> eyre::Result<()> {
|
||||
let Some(relative_path) = draft.relative_path else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let Some(kind) = draft.kind else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let size = match kind {
|
||||
RarEntryKind::File => draft
|
||||
.size
|
||||
.ok_or_else(|| eyre::eyre!("RAR file entry {relative_path} has no Size"))?,
|
||||
RarEntryKind::Directory => 0,
|
||||
};
|
||||
|
||||
entries.push(RarEntry {
|
||||
relative_path,
|
||||
kind,
|
||||
size,
|
||||
crc32: draft.crc32,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn stream_unrar_file(
|
||||
program: &Path,
|
||||
archive: &Path,
|
||||
relative_path: &str,
|
||||
frames: &mpsc::Sender<StreamInstallFrame>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> eyre::Result<()> {
|
||||
let mut child = tokio::process::Command::new(program)
|
||||
.arg("p")
|
||||
.arg("-inul")
|
||||
.arg("-cfg-")
|
||||
.arg(archive)
|
||||
.arg(relative_path)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()?;
|
||||
|
||||
let mut stdout = child
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or_eyre("unrar stdout was not captured")?;
|
||||
let mut buffer = vec![0_u8; STREAM_CHUNK_SIZE];
|
||||
|
||||
loop {
|
||||
let read = tokio::select! {
|
||||
() = cancel_token.cancelled() => {
|
||||
let _ = child.kill().await;
|
||||
eyre::bail!("streaming {relative_path} from {} was cancelled", archive.display());
|
||||
}
|
||||
read = stdout.read(&mut buffer) => read?,
|
||||
};
|
||||
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
send_stream_frame(
|
||||
frames,
|
||||
StreamInstallFrame::FileChunk {
|
||||
bytes: Bytes::copy_from_slice(&buffer[..read]),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let status = child.wait().await?;
|
||||
if !status.success() {
|
||||
eyre::bail!(
|
||||
"unrar p failed for {}:{} with status {status}",
|
||||
archive.display(),
|
||||
relative_path
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_stream_frame(
|
||||
frames: &mpsc::Sender<StreamInstallFrame>,
|
||||
frame: StreamInstallFrame,
|
||||
) -> eyre::Result<()> {
|
||||
frames
|
||||
.send(frame)
|
||||
.await
|
||||
.map_err(|_| eyre::eyre!("streamed install frame receiver closed"))
|
||||
}
|
||||
|
||||
pub fn result_line(id: &Option<Value>, command: &str, data: Value) -> eyre::Result<String> {
|
||||
output_line(json!({
|
||||
"type": "result",
|
||||
@@ -344,6 +621,57 @@ mod tests {
|
||||
assert_eq!(parsed["data"]["peer_count"], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_stream_install_command() {
|
||||
let parsed = parse_command_line(r#"{"cmd":"stream-install","game_id":"cnctw"}"#)
|
||||
.expect("command should parse");
|
||||
|
||||
assert_eq!(
|
||||
parsed.command,
|
||||
CliCommand::StreamInstall {
|
||||
game_id: "cnctw".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_unrar_technical_listing() {
|
||||
let listing = parse_unrar_listing(
|
||||
r#"
|
||||
Archive: game.eti
|
||||
Details: RAR 5
|
||||
|
||||
Name: bin/payload.bin
|
||||
Type: File
|
||||
Size: 123
|
||||
CRC32: 38B488A7
|
||||
|
||||
Name: bin
|
||||
Type: Directory
|
||||
"#,
|
||||
)
|
||||
.expect("listing should parse");
|
||||
|
||||
assert!(!listing.solid);
|
||||
assert_eq!(
|
||||
listing.entries,
|
||||
vec![
|
||||
RarEntry {
|
||||
relative_path: "bin/payload.bin".to_string(),
|
||||
kind: RarEntryKind::File,
|
||||
size: 123,
|
||||
crc32: Some(0x38B4_88A7),
|
||||
},
|
||||
RarEntry {
|
||||
relative_path: "bin".to_string(),
|
||||
kind: RarEntryKind::Directory,
|
||||
size: 0,
|
||||
crc32: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fixture_unpacker_creates_install_payload() {
|
||||
let temp = TempDir::new("lanspread-peer-cli-fixture");
|
||||
|
||||
@@ -17,6 +17,7 @@ use lanspread_peer::{
|
||||
ActiveOperation,
|
||||
ActiveOperationKind,
|
||||
InstallOperation,
|
||||
NoopStreamInstallProvider,
|
||||
PeerCommand,
|
||||
PeerEvent,
|
||||
PeerGameDB,
|
||||
@@ -24,6 +25,7 @@ use lanspread_peer::{
|
||||
PeerRuntimeHandle,
|
||||
PeerSnapshot,
|
||||
PeerStartOptions,
|
||||
StreamInstallProvider,
|
||||
migrate_legacy_state,
|
||||
start_peer_with_options,
|
||||
};
|
||||
@@ -31,6 +33,7 @@ use lanspread_peer_cli::{
|
||||
CliCommand,
|
||||
CommandEnvelope,
|
||||
DEFAULT_FIXTURE_VERSION,
|
||||
ExternalUnrarStreamProvider,
|
||||
ExternalUnrarUnpacker,
|
||||
FixtureSeed,
|
||||
FixtureUnpacker,
|
||||
@@ -134,10 +137,15 @@ async fn main() -> eyre::Result<()> {
|
||||
let (tx_events, rx_events) = mpsc::unbounded_channel();
|
||||
let peer_game_db = Arc::new(RwLock::new(PeerGameDB::new()));
|
||||
let catalog = Arc::new(RwLock::new(catalog));
|
||||
let unpacker: Arc<dyn lanspread_peer::Unpacker> = match args.unrar {
|
||||
let unrar_for_streaming = args.unrar.clone().or_else(default_unrar_program);
|
||||
let unpacker: Arc<dyn lanspread_peer::Unpacker> = match args.unrar.clone() {
|
||||
Some(path) => Arc::new(ExternalUnrarUnpacker::new(path)),
|
||||
None => Arc::new(FixtureUnpacker),
|
||||
};
|
||||
let stream_install_provider: Arc<dyn StreamInstallProvider> = match unrar_for_streaming {
|
||||
Some(path) => Arc::new(ExternalUnrarStreamProvider::new(path)),
|
||||
None => Arc::new(NoopStreamInstallProvider),
|
||||
};
|
||||
|
||||
let mut handle = start_peer_with_options(
|
||||
args.games_dir.clone(),
|
||||
@@ -148,6 +156,7 @@ async fn main() -> eyre::Result<()> {
|
||||
PeerStartOptions {
|
||||
state_dir: Some(args.state_dir.clone()),
|
||||
active_outbound_transfers: None,
|
||||
stream_install_provider: Some(stream_install_provider),
|
||||
},
|
||||
)?;
|
||||
let sender = handle.sender();
|
||||
@@ -249,6 +258,15 @@ async fn handle_command(
|
||||
})?;
|
||||
Ok(json!({"queued": true, "game_id": game_id, "install": install_after_download}))
|
||||
}
|
||||
CliCommand::StreamInstall { game_id } => {
|
||||
ensure_catalog_game(shared, game_id).await?;
|
||||
ensure_no_active_operation(shared, game_id).await?;
|
||||
let _ = game_files_for_download(sender, shared, game_id).await?;
|
||||
sender.send(PeerCommand::StreamInstallGame {
|
||||
id: game_id.clone(),
|
||||
})?;
|
||||
Ok(json!({"queued": true, "game_id": game_id}))
|
||||
}
|
||||
CliCommand::Install { game_id } => {
|
||||
ensure_catalog_game(shared, game_id).await?;
|
||||
ensure_no_active_operation(shared, game_id).await?;
|
||||
@@ -729,6 +747,15 @@ fn default_catalog_db() -> Option<PathBuf> {
|
||||
.find(|path| path.exists())
|
||||
}
|
||||
|
||||
fn default_unrar_program() -> Option<PathBuf> {
|
||||
[
|
||||
PathBuf::from("/usr/local/bin/unrar"),
|
||||
PathBuf::from("/usr/bin/unrar"),
|
||||
]
|
||||
.into_iter()
|
||||
.find(|path| path.exists())
|
||||
}
|
||||
|
||||
fn next_string(args: &mut impl Iterator<Item = OsString>, flag: &str) -> eyre::Result<String> {
|
||||
args.next()
|
||||
.ok_or_else(|| eyre::eyre!("{flag} requires a value"))?
|
||||
|
||||
Reference in New Issue
Block a user