fix(stream-install)!: stream archive payloads as raw frames
Streamed installs were sending FileChunk payloads through the shared JSON Message impl. serde_json serializes bytes as arrays of integers, which bloats wire traffic and burns CPU on large archives. Replace StreamInstallFrame encoding with tagged frames: JSON control frames keep their shape under tag 0, while file chunks carry raw bytes under tag 1. The stream install metadata now carries unpacked archive size and mandatory CRC32. The CLI unrar provider validates CRCs up front, runs one archive-wide unrar p stream, splits stdout by listed file sizes, and refuses trailing or missing bytes. That avoids solid archive re-decompression and sidesteps unrar wildcard masks for path arguments. Receivers now sample existing download progress events for streamed installs, report staging-relative chunk paths, and retry trusted peers with a fresh streamed-install transaction after a failed attempt. The current protocol policy does not preserve compatibility with older stream-install builds. Test Plan: - just fmt - just test - just clippy - git diff --check - git diff --cached --check BREAKING CHANGE: StreamInstallFrame now uses tagged frames with raw chunk payloads and requires current peers on both sides of streamed installs. Refs: NEXT_STEPS_CLAUDES_REVIEW.md
This commit is contained in:
@@ -97,11 +97,17 @@ pub enum Response {
|
||||
InternalPeerError(String),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
const STREAM_INSTALL_CONTROL_FRAME_TAG: u8 = 0;
|
||||
const STREAM_INSTALL_FILE_CHUNK_FRAME_TAG: u8 = 1;
|
||||
const STREAM_INSTALL_ENCODE_ERROR_FRAME: &[u8] =
|
||||
b"\0{\"Error\":{\"message\":\"stream install frame encoding error\"}}";
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum StreamInstallFrame {
|
||||
ArchiveBegin {
|
||||
archive_name: String,
|
||||
solid: bool,
|
||||
unpacked_size: u64,
|
||||
},
|
||||
Directory {
|
||||
relative_path: String,
|
||||
@@ -109,7 +115,7 @@ pub enum StreamInstallFrame {
|
||||
FileBegin {
|
||||
relative_path: String,
|
||||
size: u64,
|
||||
crc32: Option<u32>,
|
||||
crc32: u32,
|
||||
},
|
||||
FileChunk {
|
||||
bytes: Bytes,
|
||||
@@ -180,26 +186,59 @@ impl Message for Response {
|
||||
|
||||
impl Message for StreamInstallFrame {
|
||||
fn decode(bytes: Bytes) -> Self {
|
||||
match serde_json::from_slice(&bytes) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
tracing::error!(?e, "StreamInstallFrame decoding error");
|
||||
StreamInstallFrame::Error {
|
||||
message: format!("stream install frame decoding error: {e}"),
|
||||
}
|
||||
}
|
||||
if bytes.is_empty() {
|
||||
return stream_install_decode_error("stream install frame is empty");
|
||||
}
|
||||
|
||||
let tag = bytes[0];
|
||||
let payload = bytes.slice(1..);
|
||||
match tag {
|
||||
STREAM_INSTALL_CONTROL_FRAME_TAG => decode_stream_install_control_frame(&payload),
|
||||
STREAM_INSTALL_FILE_CHUNK_FRAME_TAG => StreamInstallFrame::FileChunk { bytes: payload },
|
||||
_ => stream_install_decode_error(format!("unknown stream install frame tag {tag}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn encode(&self) -> Bytes {
|
||||
match serde_json::to_vec(self) {
|
||||
Ok(s) => Bytes::from(s),
|
||||
Err(e) => {
|
||||
tracing::error!(?e, "StreamInstallFrame encoding error");
|
||||
Bytes::from(format!(
|
||||
r#"{{"Error": {{"message": "encoding error: {e}"}}}}"#
|
||||
))
|
||||
match self {
|
||||
StreamInstallFrame::FileChunk { bytes } => {
|
||||
tagged_stream_install_frame(STREAM_INSTALL_FILE_CHUNK_FRAME_TAG, bytes)
|
||||
}
|
||||
_ => match serde_json::to_vec(self) {
|
||||
Ok(payload) => {
|
||||
tagged_stream_install_frame(STREAM_INSTALL_CONTROL_FRAME_TAG, &payload)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(?e, "StreamInstallFrame encoding error");
|
||||
Bytes::from_static(STREAM_INSTALL_ENCODE_ERROR_FRAME)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_stream_install_control_frame(payload: &[u8]) -> StreamInstallFrame {
|
||||
match serde_json::from_slice(payload) {
|
||||
Ok(StreamInstallFrame::FileChunk { .. }) => {
|
||||
stream_install_decode_error("stream install control frame cannot contain file bytes")
|
||||
}
|
||||
Ok(frame) => frame,
|
||||
Err(e) => {
|
||||
tracing::error!(?e, "StreamInstallFrame decoding error");
|
||||
stream_install_decode_error(format!("stream install frame decoding error: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tagged_stream_install_frame(tag: u8, payload: &[u8]) -> Bytes {
|
||||
let mut frame = Vec::with_capacity(1 + payload.len());
|
||||
frame.push(tag);
|
||||
frame.extend_from_slice(payload);
|
||||
Bytes::from(frame)
|
||||
}
|
||||
|
||||
fn stream_install_decode_error(message: impl Into<String>) -> StreamInstallFrame {
|
||||
StreamInstallFrame::Error {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user