diff --git a/Cargo.lock b/Cargo.lock index aff387a..8844106 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1521,9 +1521,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1937,13 +1937,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2035,6 +2034,7 @@ name = "lanspread-peer" version = "0.1.0" dependencies = [ "bytes", + "crc32fast", "eyre", "futures", "gethostname", @@ -3103,9 +3103,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -3126,9 +3126,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -3274,9 +3274,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "s2n-codec" -version = "0.81.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d197a3c92bbe21fc00ba8366f6ba14edb8685316b6c8c14c622d3aba0a3816d8" +checksum = "a650d3f187901f3519ec8a1fe7da3faccc0b2fb40f350eda2c7851fdf2bda0f6" dependencies = [ "byteorder", "bytes", @@ -3285,9 +3285,9 @@ dependencies = [ [[package]] name = "s2n-quic" -version = "1.81.0" +version = "1.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8728244102e791769cebe44a4abace966d8826f3266e9691c4233f47921b94b8" +checksum = "c27c34127facefcd3e5530c4de5739a62cd4a593710b1194dacbd8e884b6be92" dependencies = [ "bytes", "cfg-if", @@ -3309,9 +3309,9 @@ dependencies = [ [[package]] name = "s2n-quic-core" -version = "0.81.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cc69861a4909ea508b26309504899f4b0f77bb35348f6a36b7de9a28b1a4b92" +checksum = "79fbc3f06797d985363f74de105d18554b5a272b924b166d73a6564943da1230" dependencies = [ "atomic-waker", "byteorder", @@ -3331,9 +3331,9 @@ dependencies = [ [[package]] name = "s2n-quic-crypto" -version = "0.81.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3ce7f399a87be4b49d76895cdddb987620d34f334072d011bcac913d20fe69" +checksum = "e58ea5aa39eecc29559d1e1bb4a5d55a747fa7b80cff5a3400c57489510644e3" dependencies = [ "aws-lc-rs", "cfg-if", @@ -3345,9 +3345,9 @@ dependencies = [ [[package]] name = "s2n-quic-platform" -version = "0.81.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa9004809ae3a778b8e015581a47e9fb389f9ec230456a24b81c6287b000fefe" +checksum = "4eebb6007139cfffdf3d473d39f01a214032c339432a6293b16b0f7b25343f40" dependencies = [ "cfg-if", "futures", @@ -3360,9 +3360,9 @@ dependencies = [ [[package]] name = "s2n-quic-rustls" -version = "0.81.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf7c34876c77f7560ee4385cd5ff0510acade2eb66dc237a45f7c63d2e7f1af3" +checksum = "eb0084afa65eefae2c37d9ab44118a14dfc5bb78dbf997c0f5176f7cf8d2e633" dependencies = [ "bytes", "rustls", @@ -3374,9 +3374,9 @@ dependencies = [ [[package]] name = "s2n-quic-tls" -version = "0.81.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7b14505cff3d9e39b930c31c150fe2965ee5fe1b654f7c6d33b1f50680ac0b" +checksum = "91150b25ce824ffea581b449ad04acf9b4aef2fa68a46f667cdc9cc6f7b87823" dependencies = [ "bytes", "errno", @@ -3389,9 +3389,9 @@ dependencies = [ [[package]] name = "s2n-quic-tls-default" -version = "0.81.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3297fc8531b3c19f339a3ce1969fdc0e9928cfe439ca6c9f9d9d7ca4522a3b8c" +checksum = "e1f5ae64863972facee778dc80a24317e613f035296631f267b71f225e569c22" dependencies = [ "s2n-quic-rustls", "s2n-quic-tls", @@ -3399,9 +3399,9 @@ dependencies = [ [[package]] name = "s2n-quic-transport" -version = "0.81.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ddd739c1776770dd2ab0b33da1cf372a395500252ae5250c08e2d6bf51b38f" +checksum = "3b82fca53ce1734cc1d1dca96cc9ceb65ed528f27cb43b7de865215b6cf17908" dependencies = [ "bytes", "futures-channel", @@ -5002,9 +5002,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.2" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -5107,9 +5107,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" dependencies = [ "cfg-if", "once_cell", @@ -5120,9 +5120,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" dependencies = [ "js-sys", "wasm-bindgen", @@ -5130,9 +5130,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5140,9 +5140,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" dependencies = [ "bumpalo", "proc-macro2", @@ -5153,9 +5153,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" dependencies = [ "unicode-ident", ] @@ -5209,9 +5209,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" dependencies = [ "js-sys", "wasm-bindgen", @@ -6018,18 +6018,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index c3d57d5..705d512 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ [workspace.dependencies] base64 = "0.22" bytes = { version = "1", features = ["serde"] } +crc32fast = "1" eyre = "0.6" futures = "0.3" gethostname = "1" diff --git a/NEXT_STEPS.md b/NEXT_STEPS.md new file mode 100644 index 0000000..8d734ed --- /dev/null +++ b/NEXT_STEPS.md @@ -0,0 +1,66 @@ +# Streamed Install Next Steps + +I’d treat the prototype as proof of the hard part: “can we stream +archive-derived install bytes into `local/` without making the receiver a +source?” Yes. Next I’d harden the pieces that decide whether this is +product-ready. + +1. **Done — Move from CLI-only to real app integration** + + The GUI now has an explicit “Low disk install” action in the game detail + modal for remote-only games. The Tauri backend queues that path through + `stream_install_game`, injects the shared external `unrar` stream provider, + and hands fetched file details to `StreamInstallGame` instead of the normal + download command. + +2. **Done — Replace per-file `unrar p` with a final archive provider** + + The shared external `unrar` stream provider now runs `unrar lt` once for the + archive metadata and one sequential `unrar p` pass per archive for payload + bytes. It frames directories, file starts, file chunks, and file ends from + the technical listing, so CLI and GUI callers use one purpose-built provider + instead of a per-file extraction loop. + +3. **Done — Handle solid archives deliberately** + + The provider exposes the RAR `solid` flag in `ArchiveBegin` and always uses + one sequential payload pass per archive, which is the safe path for solid + archives. S41 now verifies a real solid RAR fixture through the Docker + peer-cli flow, including local-only final state, absent root archive/sentinel, + byte count, and extracted payload SHA-256 hashes. + +4. **Done — Decide the integrity model** + + Streamed installs intentionally verify against sender archive metadata for + now: each file must match the RAR-advertised size and CRC32. That catches + transport corruption, truncation, and provider bugs, but does not claim + malicious-peer protection. Trusted content remains a separate catalog schema + step: add catalog-owned archive or extracted-file SHA-256 hashes, then verify + those at the receiver before commit. + +5. **Done — Upgrade retry/resume semantics** + + Streamed install attempts now use the same majority-validated peer set as + normal downloads, and each failed attempt rolls back its staging transaction + before trying the next peer. S42 pins the policy: retry the whole stream from + another validated peer, keep no partial files across attempts, and do not add + byte-offset resume until there is a strong reason. + +6. **Done — Expand scenario coverage** + + S43-S47 cover the remaining streamed-install edges: already-installed + rejection, corrupt archive rollback, sender disconnect mid-stream, receiver + cancel mid-stream, and multi-archive `.eti` roots streamed in sorted order. + The peer-cli harness now exposes `cancel-download` so cancellation scenarios + exercise the same runtime path as the GUI. + +7. **Done — Clean product semantics** + + The UI now keeps streamed installs in the installed visual state while making + the sharing limitation explicit: cards show `Not shareable`, and the detail + modal status shows `Installed, not shareable`. Downloaded-and-installed games + keep the normal `Installed` label. + +The remaining production-readiness step is additive: move from sender-owned RAR +metadata to catalog-owned archive or extracted-file hashes, then verify those +at the receiver before committing the streamed install. diff --git a/PEER_CLI_SCENARIOS.md b/PEER_CLI_SCENARIOS.md index a7aee4a..2bf821e 100644 --- a/PEER_CLI_SCENARIOS.md +++ b/PEER_CLI_SCENARIOS.md @@ -22,49 +22,57 @@ for deterministic local runs; mDNS/macvlan remains an environment smoke path. | S12 | Transfer serving gates | A peer has a non-catalog, missing-sentinel, active-operation, or `local/` path request. | The serving peer declines metadata/data; covered by unit tests where timing is too small for a stable CLI race test. | | S13 | Exact transferred-file equality | Repeat small and large downloads, then compare every transferred regular file against its source with SHA-256 manifests. | Source and receiver manifests match exactly for each transferred file; no extra or missing files appear in the downloaded game root. | | S14 | Large multi-peer chunked download | `fixture-alpha/alienswarm` contains a renamed RAR `.eti` larger than 100 MB. A second peer downloads it, then a third peer downloads `alienswarm` from both peers. | The third peer's downloaded files match the source by SHA-256; `download-chunk-finished` events show the large `.eti` chunks coming from both peers with byte counts balanced within one chunk. | -| S15 | Three-way version skew | Three peers advertise the same catalog game ID. Peer A has `version.ini=20250101`, peer B has `version.ini=20250201`, and peer C has `version.ini=20250301`; each version has distinguishable file contents. An empty client connects to all three and downloads the game with `install=false`. | `list-games` shows one row for the game with `peer_count=3` and `eti_game_version=20250301`. The `got-game-files` descriptor set and transfer source are peer C's newest version only; no chunks come from A or B. The receiver's `version.ini` and SHA-256 manifest match C exactly. | -| S16 | Latest-version fanout with stale peers present | Peer A has an older version of a game. Peers B and C both advertise the same newest version with matching file manifests; use a large file when proving chunk split. | The aggregated row still counts all ready peers, but eligible transfer peers are only B and C. Large-file chunks may split between B and C; peer A contributes no manifest majority vote and no file chunks. | -| S17 | Latest-version conflict rejection | Peer A has an older version. Peers B and C both advertise the newest version, but their latest-version file sizes conflict. | Validation considers only the latest-version peers, so A cannot rescue the majority. The download fails with `download-failed`, and no committed target `version.ini` remains. | +| S15 | Catalog-version skew | Three peers advertise the same catalog game ID. Peers A and B have stale `version.ini` values; peer C has the catalog's expected version. An empty client connects to all three and downloads the game with `install=false`. | `list-games` shows one row for the game with `peer_count=1` and the catalog `eti_game_version`. The `got-game-files` descriptor set and transfer source are peer C only; no chunks come from A or B. The receiver's `version.ini` and SHA-256 manifest match C exactly. | +| S16 | Catalog-version fanout with stale peers present | Peer A has a stale version of a game. Peers B and C both advertise the catalog version with matching file manifests; use a large file when proving chunk split. | The aggregated row counts only catalog-version ready peers. Large-file chunks may split between B and C; peer A is not listed as downloadable and contributes no manifest vote or file chunks. | +| S17 | Catalog-version conflict rejection | Peer A has a stale version. Peers B and C both advertise the catalog version, but their file sizes conflict. | Validation considers only the catalog-version peers, so A cannot rescue the majority. The download fails with `download-failed`, and no committed target `version.ini` remains. | | S18 | Mid-download source drop with redundancy | Client downloads a large shared game from two ready peers, then one source is killed after the download has begun. | Failed chunks are retried against the surviving source; the download finishes, no `download-failed` is emitted, and the receiver's files match the source by diff or SHA-256. | | S19 | Mid-download sole-source drop | Client downloads a large game from one source, then that source is killed after the download has begun. | The download emits `download-failed`; no committed target `version.ini` remains; any partial payload is not advertised as ready; active operation state clears so a retry is possible. | | S20 | Receiver write failure | Client downloads a large game into a constrained `/games` filesystem. | The download fails deterministically, no committed `version.ini` is advertised, and active operation state clears so the peer can retry later. | | S21 | Add-game propagation | Two connected peers are running; one peer gains a new catalog game root through a completed download or an external drop. | The other peer receives a library update without reconnecting, and `list-games` shows the new remote game under the existing peer. | | S22 | Remove-game propagation | Two connected peers are running; one peer loses a previously advertised game root. | The other peer receives a library update without dropping the peer, and `list-games` no longer shows that remote game. | -| S23 | Version bump propagation | Two connected peers are running; one peer's ready game root gets a newer `version.ini`. | The other peer receives a library update without reconnecting, and the aggregated row reflects the newer `eti_game_version`. | +| S23 | Version bump propagation | Two connected peers are running; one peer's ready game root starts with a stale `version.ini`, then changes to the catalog version. | The other peer receives a library update without reconnecting; the stale row is absent before the change, then the catalog-version game appears as downloadable. | | S24 | Two clients pull from one source | Two empty clients connect to the same source and download the same large game concurrently. | Both downloads finish, both receivers match the source by diff or SHA-256, and the source remains responsive. | | S25 | One client downloads two games concurrently | One client connected to a source issues two different `download` commands without waiting for the first to finish. | Both operations may run in parallel; both eventually finish, each game reaches the requested install state, and each transferred root matches its source. | | S26 | Same-game duplicate download rejection | A client starts downloading a game, then issues a second `download` command for the same game while the first operation is active. | The second request is rejected deterministically as an operation-in-progress condition; the first download is not corrupted and still reaches its documented final state. | | S27 | Self-connect rejection | A peer sends `connect` to its own advertised listener address. | The command fails cleanly, no self-peer entry is created, and the peer remains responsive. | | S28 | Address change without identity change | A known peer is rediscovered with the same peer ID and a different listener address while its library is still known. | The peer record updates in place to the new address, the existing library stays attached to that peer ID, and no duplicate peer entry appears. This is covered with a deterministic unit-level check until the CLI can rebind a live listener without restart. | | S29 | Empty-library peer participates | A peer with no games connects into the mesh. | Other peers list it as a peer with zero games; it can receive a download, advertise the new game without restart, and become a source. | -| S30 | 5+ peer mesh aggregation | Five peers advertise partially overlapping catalog games with a mix of unique games, shared games, and differing versions; a sixth client connects to all five. | The client shows one row per game ID, correct ready-source `peer_count`, latest `eti_game_version`, no duplicates, and no self entries. | +| S30 | 5+ peer mesh aggregation | Five peers advertise partially overlapping catalog games with a mix of unique and shared catalog-version games; a sixth client connects to all five. | The client shows one row per game ID, correct catalog-version ready-source `peer_count`, catalog `eti_game_version`, no duplicates, and no self entries. | | S31 | Bootstrapped peer becomes source in same session | An empty client downloads a game from a source, the original source shuts down, then a fresh third peer downloads the same game from the bootstrapped client. | The third peer's files match the original source by diff or SHA-256, proving downloaded files become servable without restart. | | S32 | Reinstall after uninstall | A downloaded game is installed, uninstalled, then installed again without another download. | `local/` is recreated from preserved root files, no transfer events occur during reinstall, and the game returns to `installed=true`. | | S33 | Install after external root mutation | A downloaded game root is externally mutated before `install` is issued. | The CLI fixture installer installs from the current root bytes. The resulting `local/fixture-payload.txt` must match the mutated archive bytes exactly. | | S34 | Many-small-files game without `.eti` | A catalog game root contains `version.ini` plus many small regular files and no archive. | Download with `install=false` transfers every file, chunk events are coherent for small files, and source/receiver manifests match exactly. | | S35 | Unknown game ID from remote peer | A remote peer advertises a game ID that is not in the receiver's catalog. | The receiver does not list the unknown game as downloadable, download attempts fail deterministically, and no local files are created. | -| S36 | Latest singleton beats stale majority | Five peers advertise one game; one peer has `20260501`, four peers have `20250101`. | `list-games` reports `eti_game_version=20260501`; all descriptors and chunks come from the singleton latest peer; stale peers contribute zero bytes. | +| S36 | Catalog singleton beats stale majority | Five peers advertise one game; one peer has the catalog version and four peers have stale versions. | `list-games` reports `peer_count=1` and the catalog `eti_game_version`; all descriptors and chunks come from the singleton catalog-version peer, while stale peers remain hidden and contribute zero bytes. | | S37 | Single-source download throughput | A source peer advertises a temporary catalog game with one sparse `2 GiB` `.eti`; an empty client downloads it with `install=false`. | The client emits `download-finished` with throughput measurements (`bytes`, `duration_ms`, `mib_per_s`, `mbit_per_s`), and the downloaded archive size matches the source. | | S38 | First-play launch-setting stamping | `fixture-persona/css` ships a real RAR `.eti` whose tree buries a CRLF `SmartSteamEmu.ini` with a stub `PersonaName` line under `engine/bin/win64/steam_settings/`, plus a stub `account_name.txt` and `language.txt` under `profiles/local/`. A peer installs `css` (with `--unrar`), then sends `play css` with a username and language, then `play css` again. | After install the marker `games/css/launch_settings_applied` is absent and the stub files are intact under `local/`. The first `play` returns `already_applied=false` with `account_name_written`, `language_written`, and `persona_name_written` all true; the deep `SmartSteamEmu.ini` `PersonaName` value becomes the username with its `\r\n` ending and sibling lines preserved, `account_name.txt` becomes the username, `language.txt` becomes the passed language, and the marker now exists. A second `play` returns `already_applied=true`, rewrites nothing, and leaves the files untouched even if their values were reset externally. | +| S39 | Streamed install without keeping archive payload | Empty client connects to `fixture-bravo`, then sends `stream-install cnctw`. The source has real RAR `.eti` payload entries under `bin/` and `data/`; the receiver uses the container-bundled `unrar` stream provider. | Client emits `download-begin`, streamed `download-chunk-finished`, `download-finished`, `install-begin`, and `install-finished`. Local `cnctw` is `downloaded=false`, `installed=true`, `availability=LocalOnly`; root `version.ini` and `.eti` are absent; `local/bin/cnctw-payload.bin` and `local/data/cnctw-assets.dat` match `unrar p` output by SHA-256; the source reports no active outbound transfer for `cnctw` after completion. | +| S40 | Streamed install receiver is not a peer source | After S39, a third peer connects only to the streamed-install receiver. | The third peer may see the receiver's local-only summary in peer snapshots, but `list-games` remote aggregation does not expose `cnctw` as downloadable, `peer_count` remains zero/absent, and attempting `download cnctw` fails with no local files created. | +| S41 | Solid archive streamed install | Empty client connects to a peer serving `fixture-solid/cnctw`, whose `.eti` is a real solid RAR archive. The receiver uses the container-bundled `unrar` stream provider. | The fixture is verified as solid with `unrar lt`; streamed install finishes with `downloaded=false`, `installed=true`, `availability=LocalOnly`; root archive and `version.ini` are absent; streamed byte count equals the extracted solid entries; local payload SHA-256 hashes match `unrar p` output. | +| S42 | Streamed install whole-stream retry | Empty client connects to two peers serving the same catalog-version `cnctw`: one broken source whose `--unrar` path is missing, followed by one good source. | The broken source sorts before the good source in retry order, contributes zero chunks, and the good source completes a fresh whole-stream attempt. The final state is local-only installed, no root archive/sentinel, no `.local.installing`, byte count matches the extracted entries, and payload hashes match the good source. | +| S43 | Already-installed streamed install rejection | A client first stream-installs `cnctw`, then attempts `stream-install cnctw` again. | The second request emits `download-failed`, does not emit a new success event, leaves the existing local-only install intact, and clears active operations. | +| S44 | Corrupt archive streamed install rollback | A source advertises catalog-version `cnctw`, but its root `.eti` is replaced with invalid bytes before the client requests `stream-install cnctw`. | The stream emits `download-failed`, does not emit download/install success, clears active operations, and leaves no `local/`, `.local.installing`, root archive, or root `version.ini` on the receiver. | +| S45 | Sender disconnect during streamed install | A source serves large catalog-version `alienswarm`; after the client receives the first streamed chunk, the source container is killed. | The operation reaches a terminal failure/peers-gone event, emits no download/install success, clears active operations, and rolls back local/staging state. | +| S46 | Receiver cancel during streamed install | A client starts streaming large catalog-version `alienswarm`, receives the first chunk, then sends `cancel-download alienswarm`. | The receiver cancels without emitting download/install success or a user-visible download failure, clears active operations, and rolls back local/staging state. | +| S47 | Multi-archive streamed install order | A source serves `fixture-multi/cnctw` with two root `.eti` archives named to require sorted processing. | Streamed chunk paths arrive in root archive sort order, both payloads install under `local/`, the receiver is local-only installed, and no root archives or sentinel are committed. | ## Version-Skew Contract -Use S15-S17 to pin down what "newer" means when several peers have the same -game ID: +Use S15-S17 to pin down what happens when several peers have the same game ID +but only some match the local catalog version: -- Version comparison uses the eight-digit `version.ini` string, so use sortable - `YYYYMMDD` values in manual fixtures. +- The receiver's catalog is authoritative. A remote root whose `version.ini` + does not match the catalog's expected version for that game ID is not + downloadable. - `list-games` aggregates by game ID. The game appears once; `peer_count` - counts all ready peers with that ID, including peers that only have older - versions. -- The aggregated `eti_game_version` must be the newest ready version. + counts only ready peers with that ID and the catalog version. +- The aggregated `eti_game_version` must be the catalog version. - The descriptor set emitted to the download path, file-size validation, and - transfer planning are latest-only. Older-version peers may be queried by a - generic detail request, but their descriptors must not supply download - descriptors, majority votes, or chunks once a newer version exists. -- If exactly one peer has the latest version, that peer is the only transfer - source. If several peers tie on the latest version, validation and chunk - fanout happen among that latest-version set only. + transfer planning are catalog-version-only. Stale peers must not supply + download descriptors, majority votes, or chunks. +- If exactly one peer has the catalog version, that peer is the only transfer + source. If several peers match the catalog version, validation and chunk + fanout happen among that catalog-version set only. - Capture proof with the `list-games` row, `got-game-files` descriptors, `download-chunk-finished` source addresses, and source/receiver SHA-256 manifests. @@ -78,7 +86,7 @@ GUI: payload files may remain, but they must not be advertised as a ready local game and must not leave an active operation stuck. - Source failure during a redundant download should retry failed chunks against - another validated source for the same latest-version file. + another validated source for the same catalog-version file. - Live local library changes are observable by connected peers through library deltas; reconnect is not required for add, remove, or version-bump cases. - Same-game operations are single-flight. A duplicate download request while a @@ -87,10 +95,10 @@ GUI: are not downloadable. For a manual run, prefer a catalog game ID already served by the fixture lab, -such as `cnc4`, then create temporary `just peer-cli-run` game roots with -different `version.ini` contents. The existing alpha/bravo/charlie fixtures -cover duplicate-source and shared-game cases, but not the three-version skew -until a dedicated fixture or temporary games root is prepared. +such as `cnc4`, then create temporary `just peer-cli-run` game roots where some +peers match the catalog version and others deliberately use stale +`version.ini` contents. The existing alpha/bravo/charlie fixtures cover +duplicate-source and shared-game cases; S15-S17 add the focused skew cases. ## First-Play Launch-Setting Contract @@ -105,13 +113,147 @@ Use S38 to pin down how launcher settings are stamped into an installed game: line keeps its existing line ending (`\n` or `\r\n`). - The marker records only that we *tried*: it is written unconditionally after the first play, so a game with none of these files is still marked done. -- S38 needs a real archive expanded with `--unrar`, so it runs against the host - `lanspread-peer-cli` binary rather than the Docker matrix image (which omits - `unrar`). The peer crate's `launch_settings` unit tests cover the rewrite, - line-ending, and marker logic deterministically. +- S38 needs a real archive expanded with `--unrar`; the Docker matrix image now + carries the Linux sidecar for streamed-install coverage, while the peer + crate's `launch_settings` unit tests cover the rewrite, line-ending, and + marker logic deterministically. + +## Streamed Install Archive Contract + +Use S39-S41 to pin down low-disk streamed installs: + +- The stream provider performs one archive metadata pass and one payload pass + per `.eti`, then frames entry boundaries for the receiver. +- Non-solid and solid archives both install into `local/` without committing a + root archive or root `version.ini`, so the receiver is installed but not a + downloadable source. +- Streamed install integrity is currently sender archive integrity: size and + RAR CRC32 must match the sender's archive metadata. The SHA-256 checks in the + scenarios prove the Docker/provider path matches the source fixture; they are + not catalog-owned trust anchors. +- S41 verifies the fixture is actually solid inside the source container, so + solid handling stays covered by the same Docker harness as the existing + streamed-install scenarios. +- S42 verifies retry/resume semantics: failed streamed attempts roll back their + staging directory and retry the whole stream from another validated peer. + There is no byte-offset resume contract. +- S43-S47 cover the remaining streamed-install failure and archive-shape edges: + already-installed rejection, corrupt archive rollback, sender disconnect, + receiver cancel, and multi-archive root sorting. ## Run Log +### 2026-06-07 - Catalog-Version Matrix Alignment (S1-S47) + +- Code under test aligned checked-in fixture `version.ini` sentinels with the + catalog, made `run_extended_scenarios.py` stamp generated fixture games with + catalog versions by default, updated S15-S17/S23/S30/S36/S37 to assert + catalog-authoritative aggregation, and wired S38 into the executable matrix. +- Gates before Docker: `python3 -m py_compile + crates/lanspread-peer-cli/scripts/run_extended_scenarios.py` passed. +- Targeted rebuilt-image runner: + `python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py S3 S8 S14 S15 S16 S17 S21 S22 S23 S24 S29 S30 S31 S34 S36 S37 S39 S40 S41 S42 S43 S44 S45 S46 S47 --build-image` + passed. +- S38 standalone runner: + `python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py S38` + passed, proving the real-RAR `css` fixture installs with the container + `/usr/local/bin/unrar` sidecar and stamps launch settings only once. +- Full matrix runner: + `python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py` + passed for S1-S47 against the rebuilt `lanspread-peer-cli:dev` image. +- The final full-run highlights included S3 aggregation, S15-S17 + catalog-version skew/fanout/conflict, S23 stale-to-catalog propagation, S30 + mesh aggregation, S36 catalog singleton over stale majority, S37 throughput, + S38 first-play stamping, and S39-S47 streamed-install coverage. + +### 2026-06-07 - Streamed Install Edge Coverage (S43-S47) + +- Code under test added `cancel-download` to `lanspread-peer-cli`, added the + tiny `fixture-multi/cnctw` two-archive fixture, and added S43-S47 in + `run_extended_scenarios.py`. +- Gates before Docker: `just fmt` and `python3 -m py_compile + crates/lanspread-peer-cli/scripts/run_extended_scenarios.py` passed. +- Runner: + `python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py S43 S44 S45 S46 S47 --build-image` + passed against the rebuilt `lanspread-peer-cli:dev` image. +- S43 stream-installed `cnctw`, retried `stream-install cnctw`, observed + `download-failed`, and verified the existing local-only install stayed intact. +- S44 replaced the source `cnctw.eti` with invalid bytes. The receiver emitted + `download-failed`, cleared active operations, and left no `local/`, + `.local.installing`, root archive, or root `version.ini`. +- S45 killed the sole `alienswarm` source after the first streamed chunk. The + receiver ended with `download-failed`, emitted no success, cleared active + operations, and rolled back local/staging state. +- S46 cancelled `alienswarm` on the receiver after the first streamed chunk. + The receiver emitted no success and no user-visible `download-failed`, cleared + active operations, and rolled back local/staging state. +- S47 streamed `fixture-multi/cnctw` and observed chunk paths in sorted root + archive order: `cnctw/.local.installing/order/first.txt`, then + `cnctw/.local.installing/order/second.txt`. + +### 2026-06-07 - Streamed Install Whole-Stream Retry (S42) + +- Code under test added S42 in `run_extended_scenarios.py`. +- Gates before Docker: `python3 -m py_compile + crates/lanspread-peer-cli/scripts/run_extended_scenarios.py` passed. +- Runner: + `python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py S42` + passed against the current `lanspread-peer-cli:dev` image. +- S42 started a broken source with `--unrar /missing-unrar` and a good source + with the same catalog-version `cnctw` metadata. The broken source sorted first + (`10.66.0.2:32897`) and the good source second (`10.66.0.3:34092`). +- The broken source contributed zero chunks; the good source completed the fresh + whole-stream attempt with `3145728` streamed file bytes. +- The final client state was `downloaded=false`, `installed=true`, + `availability=LocalOnly`, with no root `version.ini`, no root `cnctw.eti`, + and no `.local.installing` staging directory. Payload SHA-256 hashes matched + the good source's `unrar p` output. + +### 2026-06-07 - Solid Streamed Install Coverage (S41) + +- Code under test added `fixture-solid/cnctw`, a real solid RAR `.eti`, plus + S41 in `run_extended_scenarios.py`. +- Gates before Docker: `just fmt`, `git diff --check`, and + `python3 -m py_compile crates/lanspread-peer-cli/scripts/run_extended_scenarios.py` + passed. +- Runner: + `python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py S41 --build-image` + passed against the rebuilt `lanspread-peer-cli:dev` image. +- S41 verified the source archive with `unrar lt -cfg-` inside the source + container; the archive reported `Details: RAR 5, solid`. +- The streamed install finished with `downloaded=false`, `installed=true`, + `availability=LocalOnly`, no root `version.ini`, and no root `cnctw.eti`. +- The client received `118` streamed file bytes, matching the extracted solid + entries. Payload SHA-256 hashes matched `unrar p` output: + `88764c9a6c9b5b846b4323cf7725cb7fd70766ddd7fba4168332804a839fa193` + (`bin/cnctw-solid-payload.bin`) and + `44afc308269b2381b7c707a056dd8d9d393274108ac4d880237fa6772c861d7a` + (`data/cnctw-solid-assets.dat`). + +### 2026-06-07 - Streamed Install Prototype (S39-S40) + +- Code under test added `stream-install` to `lanspread-peer-cli`, a peer + `StreamInstallGame` command, streamed install frames over QUIC, and an + injected `unrar lt`/`unrar p` provider for archive-derived bytes. +- Gates before Docker: `just fmt` and + `RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= just test` passed for the + workspace. +- Runner: + `python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py S39 S40 --build-image` + passed against the rebuilt `lanspread-peer-cli:dev` image. +- S39 streamed a catalog-version-adjusted `cnctw` fixture from a real RAR + `.eti` into the receiver's `local/` only. The receiver had + `downloaded=false`, `installed=true`, `availability=LocalOnly`, no root + `version.ini`, no root `.eti`, and payload SHA-256 hashes + `82f4da22dc042166def2a5ee2eca19fc9e52785f99838e86c32167cb342e2588` + (`bin/cnctw-payload.bin`) and + `abf833a06c74ea9f17d505c2684186491898ce906405e0f098f0deac19476b06` + (`data/cnctw-assets.dat`) matching `unrar p`. +- S40 connected an observer only to that streamed-install receiver. The + observer saw the receiver's `cnctw` summary as local-only, remote aggregation + hid it as a downloadable source, and `download cnctw` failed with + `no peers have game cnctw`. + ### 2026-05-28 - First-Play Launch-Setting Stamping (S38) - Code under test moved the `account_name.txt`/`language.txt` overwrite out of diff --git a/crates/lanspread-peer-cli/Dockerfile b/crates/lanspread-peer-cli/Dockerfile index ca24bfc..ad636f3 100644 --- a/crates/lanspread-peer-cli/Dockerfile +++ b/crates/lanspread-peer-cli/Dockerfile @@ -4,14 +4,16 @@ WORKDIR /work COPY . . RUN cargo build --release -p lanspread-peer-cli -FROM debian:bookworm-slim +FROM debian:trixie-slim RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates \ + && apt-get install -y --no-install-recommends ca-certificates libstdc++6 \ && rm -rf /var/lib/apt/lists/* COPY --from=build /work/target/release/lanspread-peer-cli /usr/local/bin/lanspread-peer-cli COPY crates/lanspread-tauri-deno-ts/src-tauri/game.db /app/game.db +COPY crates/lanspread-tauri-deno-ts/src-tauri/binaries/unrar-x86_64-unknown-linux-gnu /usr/local/bin/unrar +RUN chmod +x /usr/local/bin/unrar ENTRYPOINT ["lanspread-peer-cli"] CMD ["--games-dir", "/games", "--state-dir", "/state", "--catalog-db", "/app/game.db"] diff --git a/crates/lanspread-peer-cli/README.md b/crates/lanspread-peer-cli/README.md index 6204cce..8a5d0a0 100644 --- a/crates/lanspread-peer-cli/README.md +++ b/crates/lanspread-peer-cli/README.md @@ -42,3 +42,7 @@ echoed back on the result or error line. {"id":"u1","cmd":"uninstall","game_id":"fixture-one"} {"id":"q1","cmd":"shutdown"} ``` + +The `status` result includes receiver-side `active_operations` and +sender-side `active_outbound_transfers` counts by game ID, which the scenario +runner uses to verify transfer lifecycle cleanup. diff --git a/crates/lanspread-peer-cli/fixtures/fixture-alpha/alienswarm/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-alpha/alienswarm/version.ini index fc6c4b4..902c03e 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-alpha/alienswarm/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-alpha/alienswarm/version.ini @@ -1 +1 @@ -20250101 +20190317 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-alpha/bf1942/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-alpha/bf1942/version.ini index 39a15df..1fb9726 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-alpha/bf1942/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-alpha/bf1942/version.ini @@ -1 +1 @@ -20250103 +20160130 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-alpha/ggoo/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-alpha/ggoo/version.ini index dc5f751..787e406 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-alpha/ggoo/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-alpha/ggoo/version.ini @@ -1 +1 @@ -20250102 +20200721 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-bravo/bfbc2/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-bravo/bfbc2/version.ini index 50ae903..7bef19b 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-bravo/bfbc2/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-bravo/bfbc2/version.ini @@ -1 +1 @@ -20250201 +20210416 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-bravo/cnc4/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-bravo/cnc4/version.ini index ad51826..8ceaed4 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-bravo/cnc4/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-bravo/cnc4/version.ini @@ -1 +1 @@ -20250202 +20170204 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-bravo/cnctw/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-bravo/cnctw/version.ini index 7f9b55c..f71f081 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-bravo/cnctw/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-bravo/cnctw/version.ini @@ -1 +1 @@ -20250203 +20160128 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-bravo/ggoo/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-bravo/ggoo/version.ini index dc5f751..787e406 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-bravo/ggoo/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-bravo/ggoo/version.ini @@ -1 +1 @@ -20250102 +20200721 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-charlie/cnc4/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-charlie/cnc4/version.ini index ad51826..8ceaed4 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-charlie/cnc4/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-charlie/cnc4/version.ini @@ -1 +1 @@ -20250202 +20170204 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-charlie/cod5/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-charlie/cod5/version.ini index a607fb0..e3a4855 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-charlie/cod5/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-charlie/cod5/version.ini @@ -1 +1 @@ -20250301 +20160920 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-charlie/cod6/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-charlie/cod6/version.ini index 08a14c7..1ddcbf2 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-charlie/cod6/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-charlie/cod6/version.ini @@ -1 +1 @@ -20250302 +20200315 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-charlie/coh/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-charlie/coh/version.ini index b12838a..940d96b 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-charlie/coh/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-charlie/coh/version.ini @@ -1 +1 @@ -20250303 +20200907 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/a-first.eti b/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/a-first.eti new file mode 100644 index 0000000..3377feb Binary files /dev/null and b/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/a-first.eti differ diff --git a/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/version.ini new file mode 100644 index 0000000..f71f081 --- /dev/null +++ b/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/version.ini @@ -0,0 +1 @@ +20160128 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/z-second.eti b/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/z-second.eti new file mode 100644 index 0000000..ac694b4 Binary files /dev/null and b/crates/lanspread-peer-cli/fixtures/fixture-multi/cnctw/z-second.eti differ diff --git a/crates/lanspread-peer-cli/fixtures/fixture-persona/css/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-persona/css/version.ini index 0cca4ac..c594e78 100644 --- a/crates/lanspread-peer-cli/fixtures/fixture-persona/css/version.ini +++ b/crates/lanspread-peer-cli/fixtures/fixture-persona/css/version.ini @@ -1 +1 @@ -20250101 \ No newline at end of file +20240623 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/fixtures/fixture-solid/cnctw/cnctw.eti b/crates/lanspread-peer-cli/fixtures/fixture-solid/cnctw/cnctw.eti new file mode 100644 index 0000000..ac98efd Binary files /dev/null and b/crates/lanspread-peer-cli/fixtures/fixture-solid/cnctw/cnctw.eti differ diff --git a/crates/lanspread-peer-cli/fixtures/fixture-solid/cnctw/version.ini b/crates/lanspread-peer-cli/fixtures/fixture-solid/cnctw/version.ini new file mode 100644 index 0000000..f71f081 --- /dev/null +++ b/crates/lanspread-peer-cli/fixtures/fixture-solid/cnctw/version.ini @@ -0,0 +1 @@ +20160128 \ No newline at end of file diff --git a/crates/lanspread-peer-cli/scripts/run_extended_scenarios.py b/crates/lanspread-peer-cli/scripts/run_extended_scenarios.py index 8c262a6..1a6b672 100644 --- a/crates/lanspread-peer-cli/scripts/run_extended_scenarios.py +++ b/crates/lanspread-peer-cli/scripts/run_extended_scenarios.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 -"""Run the peer-cli scenarios S1-S36 through Docker.""" +"""Run the peer-cli scenarios S1-S47 through Docker.""" from __future__ import annotations import argparse import hashlib +import ipaddress import json import os import queue +import shlex import shutil import subprocess import sys @@ -26,7 +28,20 @@ CONTAINER_PREFIX = "lanspread-peer-cli-ext" CATALOG_DB = "/app/game.db" FIXTURES = REPO / "crates" / "lanspread-peer-cli" / "fixtures" CHUNK_SIZE = 128 * 1024 * 1024 +CATALOG_VERSIONS = { + "alienswarm": "20190317", + "bf1942": "20160130", + "bfbc2": "20210416", + "cnc4": "20170204", + "cnctw": "20160128", + "cod5": "20160920", + "cod6": "20200315", + "coh": "20200907", + "css": "20240623", + "ggoo": "20200721", +} PERF_GAME_ID = "bf1942" +PERF_GAME_VERSION = CATALOG_VERSIONS[PERF_GAME_ID] PERF_GAME_SIZE = 2 * 1024 * 1024 * 1024 IGNORED_DIFF_NAMES = {".lanspread", ".lanspread.json", "local"} @@ -303,8 +318,8 @@ class Runner: ("S13", self.s13_exact_transfer_equality), ("S14", self.s14_large_multi_peer_chunking), ("S15", self.s15_three_way_version_skew), - ("S16", self.s16_latest_fanout_with_stale), - ("S17", self.s17_latest_conflict_rejection), + ("S16", self.s16_catalog_fanout_with_stale), + ("S17", self.s17_catalog_conflict_rejection), ("S18", self.s18_redundant_source_drop), ("S19", self.s19_sole_source_drop), ("S20", self.s20_receiver_write_failure), @@ -323,8 +338,18 @@ class Runner: ("S33", self.s33_install_after_mutation), ("S34", self.s34_many_small_files), ("S35", self.s35_unknown_game_filtered), - ("S36", self.s36_latest_singleton), + ("S36", self.s36_catalog_singleton), ("S37", self.s37_single_source_download_throughput), + ("S38", self.s38_first_play_launch_settings), + ("S39", self.s39_streamed_install_local_only), + ("S40", self.s40_streamed_receiver_not_source), + ("S41", self.s41_solid_archive_streamed_install), + ("S42", self.s42_streamed_install_retries_next_source), + ("S43", self.s43_streamed_install_rejects_installed_game), + ("S44", self.s44_corrupt_stream_rolls_back), + ("S45", self.s45_sender_disconnect_mid_stream), + ("S46", self.s46_receiver_cancel_mid_stream), + ("S47", self.s47_multi_archive_streams_in_sorted_order), ] for scenario_id, scenario in scenarios: @@ -509,20 +534,20 @@ class Runner: def s8_ambiguous_metadata_rejection(self) -> str: dir_a = self.fixture_root / "s8-a" dir_b = self.fixture_root / "s8-b" - copy_game("ggoo", dir_a, version="20260101") - copy_game("ggoo", dir_b, version="20260101") + copy_game("ggoo", dir_a) + copy_game("ggoo", dir_b) with (dir_b / "ggoo" / "ggoo.eti").open("ab") as handle: handle.write(b"conflict") peer_a = self.peer("s8-a", games_dir=dir_a) peer_b = self.peer("s8-b", games_dir=dir_b) client = self.peer("s8-client") connect_many(client, [peer_a, peer_b]) - wait_remote_game(client, "ggoo", peer_count=2, version="20260101") + wait_remote_game(client, "ggoo", peer_count=2, version=CATALOG_VERSIONS["ggoo"]) waiter = LineWaiter(len(client.output)) client.send({"cmd": "download", "game_id": "ggoo", "install": False}) client.wait_for(event_is("download-failed", "ggoo"), timeout=30, description="ggoo failed", waiter=waiter) assert_not_exists(client.host_games_dir / "ggoo" / "version.ini") - return "conflicting latest ggoo file sizes emitted download-failed and left no version.ini" + return "conflicting catalog-version ggoo file sizes emitted download-failed and left no version.ini" def s9_missing_game(self) -> str: client = self.peer("s9-client") @@ -604,7 +629,7 @@ class Runner: diff_game_dirs(source_dir / game_id, stage.host_games_dir / game_id) client = self.peer("s14-client") connect_many(client, [alpha, stage]) - wait_remote_game(client, game_id, peer_count=2, version="20260520") + wait_remote_game(client, game_id, peer_count=2, version=PERF_GAME_VERSION) waiter = LineWaiter(len(client.output)) client.send({"cmd": "download", "game_id": game_id, "install": False}) client.wait_for(event_is("download-finished", game_id), timeout=90, description="client finish", waiter=waiter) @@ -618,7 +643,11 @@ class Runner: return f"{game_id} downloaded from two sources, diff matched, chunk totals={totals}" def s15_three_way_version_skew(self) -> str: - specs = [("s15-a", "20250101"), ("s15-b", "20250201"), ("s15-c", "20250301")] + specs = [ + ("s15-a", "20150101"), + ("s15-b", "20160101"), + ("s15-c", CATALOG_VERSIONS["cnc4"]), + ] peers = [] for name, version in specs: game_dir = self.fixture_root / name @@ -626,19 +655,19 @@ class Runner: peers.append(self.peer(name, games_dir=game_dir)) client = self.peer("s15-client") connect_many(client, peers) - wait_remote_game(client, "cnc4", peer_count=3, version="20250301") + wait_remote_game(client, "cnc4", peer_count=1, version=CATALOG_VERSIONS["cnc4"]) waiter = LineWaiter(len(client.output)) client.send({"cmd": "download", "game_id": "cnc4", "install": False}) client.wait_for(event_is("download-finished", "cnc4"), timeout=60, description="cnc4 finish", waiter=waiter) assert_only_chunk_sources(client, "cnc4", {peers[2].ready_addr}) diff_game_dirs(peers[2].host_games_dir / "cnc4", client.host_games_dir / "cnc4") - return "three-way skew selected only 20250301 peer and receiver diffed cleanly" + return "three-way skew exposed only the catalog-version peer and receiver diffed cleanly" - def s16_latest_fanout_with_stale(self) -> str: + def s16_catalog_fanout_with_stale(self) -> str: specs = [ - ("s16-a", "20250101"), - ("s16-b", "20250301"), - ("s16-c", "20250301"), + ("s16-a", "20180101"), + ("s16-b", CATALOG_VERSIONS["alienswarm"]), + ("s16-c", CATALOG_VERSIONS["alienswarm"]), ] peers = [] for name, version in specs: @@ -647,7 +676,7 @@ class Runner: peers.append(self.peer(name, games_dir=game_dir)) client = self.peer("s16-client") connect_many(client, peers) - wait_remote_game(client, "alienswarm", peer_count=3, version="20250301") + wait_remote_game(client, "alienswarm", peer_count=2, version=CATALOG_VERSIONS["alienswarm"]) waiter = LineWaiter(len(client.output)) client.send({"cmd": "download", "game_id": "alienswarm", "install": False}) client.wait_for(event_is("download-finished", "alienswarm"), timeout=90, description="alienswarm finish", waiter=waiter) @@ -656,13 +685,13 @@ class Runner: if peers[0].ready_addr in totals: raise ScenarioError(f"stale peer contributed chunks: {totals}") diff_game_dirs(peers[1].host_games_dir / "alienswarm", client.host_games_dir / "alienswarm") - return f"latest B/C peers served alienswarm while stale A contributed zero; totals={totals}" + return f"catalog-version B/C peers served alienswarm while stale A contributed zero; totals={totals}" - def s17_latest_conflict_rejection(self) -> str: + def s17_catalog_conflict_rejection(self) -> str: specs = [ - ("s17-a", "20250101", False), - ("s17-b", "20250301", False), - ("s17-c", "20250301", True), + ("s17-a", "20150101", False), + ("s17-b", CATALOG_VERSIONS["cnc4"], False), + ("s17-c", CATALOG_VERSIONS["cnc4"], True), ] peers = [] for name, version, conflict in specs: @@ -674,12 +703,12 @@ class Runner: peers.append(self.peer(name, games_dir=game_dir)) client = self.peer("s17-client") connect_many(client, peers) - wait_remote_game(client, "cnc4", peer_count=3, version="20250301") + wait_remote_game(client, "cnc4", peer_count=2, version=CATALOG_VERSIONS["cnc4"]) waiter = LineWaiter(len(client.output)) client.send({"cmd": "download", "game_id": "cnc4", "install": False}) client.wait_for(event_is("download-failed", "cnc4"), timeout=30, description="cnc4 failed", waiter=waiter) assert_not_exists(client.host_games_dir / "cnc4" / "version.ini") - return "latest-version file conflict failed download and left no committed version.ini" + return "catalog-version file conflict failed download and left no committed version.ini" def s18_redundant_source_drop(self) -> str: source_a_dir = self.fixture_root / "s18-a" @@ -762,13 +791,13 @@ class Runner: def s23_version_bump_propagation(self) -> str: alpha = self.peer("s23-alpha") bravo_dir = self.fixture_root / "s23-bravo" - copy_game("cnc4", bravo_dir, version="20250101") + copy_game("cnc4", bravo_dir, version="20160101") bravo = self.peer("s23-bravo", games_dir=bravo_dir) connect_many(alpha, [bravo]) - wait_remote_game(alpha, "cnc4", peer_count=1, version="20250101") - (bravo_dir / "cnc4" / "version.ini").write_text("20260501", encoding="utf-8") - wait_remote_game(alpha, "cnc4", peer_count=1, version="20260501") - return "alpha observed cnc4 eti_game_version change 20250101 -> 20260501 without reconnect" + wait_remote_absent(alpha, "cnc4", timeout=5) + (bravo_dir / "cnc4" / "version.ini").write_text(CATALOG_VERSIONS["cnc4"], encoding="utf-8") + wait_remote_game(alpha, "cnc4", peer_count=1, version=CATALOG_VERSIONS["cnc4"]) + return "alpha observed stale cnc4 become catalog-version downloadable without reconnect" def s24_two_clients_one_source(self) -> str: source = self.peer("s24-alpha", games_dir=FIXTURES / "fixture-alpha", readonly_games=True) @@ -865,11 +894,11 @@ class Runner: def s30_mesh_aggregation(self) -> str: dirs = [] specs = [ - ("s30-a", [("ggoo", "20250101"), ("bf1942", "20250101")]), - ("s30-b", [("ggoo", "20250101"), ("cnc4", "20250101")]), - ("s30-c", [("cnc4", "20250301"), ("cod5", "20250101")]), - ("s30-d", [("cnctw", "20250101"), ("coh", "20250101")]), - ("s30-e", [("cnctw", "20250201"), ("bf1942", "20250201")]), + ("s30-a", [("ggoo", CATALOG_VERSIONS["ggoo"]), ("bf1942", CATALOG_VERSIONS["bf1942"])]), + ("s30-b", [("ggoo", CATALOG_VERSIONS["ggoo"]), ("cnc4", CATALOG_VERSIONS["cnc4"])]), + ("s30-c", [("cnc4", CATALOG_VERSIONS["cnc4"]), ("cod5", CATALOG_VERSIONS["cod5"])]), + ("s30-d", [("cnctw", CATALOG_VERSIONS["cnctw"]), ("coh", CATALOG_VERSIONS["coh"])]), + ("s30-e", [("cnctw", CATALOG_VERSIONS["cnctw"]), ("bf1942", CATALOG_VERSIONS["bf1942"])]), ] peers = [] for name, games in specs: @@ -881,12 +910,12 @@ class Runner: client = self.peer("s30-client") connect_many(client, peers) expected = { - "ggoo": (2, "20250101"), - "bf1942": (2, "20250201"), - "cnc4": (2, "20250301"), - "cod5": (1, "20250101"), - "cnctw": (2, "20250201"), - "coh": (1, "20250101"), + "ggoo": (2, CATALOG_VERSIONS["ggoo"]), + "bf1942": (2, CATALOG_VERSIONS["bf1942"]), + "cnc4": (2, CATALOG_VERSIONS["cnc4"]), + "cod5": (1, CATALOG_VERSIONS["cod5"]), + "cnctw": (2, CATALOG_VERSIONS["cnctw"]), + "coh": (1, CATALOG_VERSIONS["coh"]), } for game_id, (peer_count, version) in expected.items(): wait_remote_game(client, game_id, peer_count=peer_count, version=version) @@ -896,7 +925,7 @@ class Runner: raise ScenarioError(f"duplicate game rows: {ids}") if any(peer["peer_id"] == client.peer_id for peer in client.list_peers()): raise ScenarioError("client listed itself as a peer") - return f"client aggregated {len(expected)} IDs from 5 peers with expected peer_count/latest versions" + return f"client aggregated {len(expected)} IDs from 5 peers with expected peer_count/catalog versions" def s31_bootstrapped_peer_source(self) -> str: source = self.peer("s31-alpha", games_dir=FIXTURES / "fixture-alpha", readonly_games=True) @@ -992,34 +1021,34 @@ class Runner: assert_not_exists(client.host_games_dir / "mystery-game") return f"unknown game absent from list-games; download errored '{err['error']}'; no local files" - def s36_latest_singleton(self) -> str: + def s36_catalog_singleton(self) -> str: peers = [] for index in range(5): game_dir = self.fixture_root / f"s36-{index}" - version = "20260501" if index == 0 else "20250101" + version = CATALOG_VERSIONS["cnc4"] if index == 0 else "20160101" copy_game("cnc4", game_dir, version=version) peers.append(self.peer(f"s36-{index}", games_dir=game_dir)) client = self.peer("s36-client") connect_many(client, peers) - wait_remote_game(client, "cnc4", peer_count=5, version="20260501") + wait_remote_game(client, "cnc4", peer_count=1, version=CATALOG_VERSIONS["cnc4"]) waiter = LineWaiter(len(client.output)) client.send({"cmd": "download", "game_id": "cnc4", "install": False}) got = client.wait_for(event_is("got-game-files", "cnc4"), timeout=20, description="got game files", waiter=waiter) client.wait_for(event_is("download-finished", "cnc4"), timeout=60, description="download finish", waiter=waiter) - latest_addr = peers[0].ready_addr - if latest_addr is None: - raise ScenarioError("latest peer had no ready addr") + catalog_addr = peers[0].ready_addr + if catalog_addr is None: + raise ScenarioError("catalog-version peer had no ready addr") for item in client.output: if item.get("type") != "event" or item.get("event") != "download-chunk-finished": continue data = item["data"] - if data.get("game_id") == "cnc4" and data.get("peer_addr") != latest_addr: + if data.get("game_id") == "cnc4" and data.get("peer_addr") != catalog_addr: raise ScenarioError(f"stale peer contributed chunk: {data}") diff_game_dirs(peers[0].host_games_dir / "cnc4", client.host_games_dir / "cnc4") descs = got["data"]["file_descriptions"] if not descs: raise ScenarioError("got-game-files had no descriptors") - return "client reported latest 20260501 with peer_count=5; only singleton latest peer sent chunks; diff matched" + return "client reported singleton catalog-version peer; stale peers stayed hidden and sent no chunks; diff matched" def s37_single_source_download_throughput(self) -> str: source_dir = self.fixture_root / "s37-source" @@ -1027,7 +1056,7 @@ class Runner: source = self.peer("s37-source", games_dir=source_dir) client = self.peer("s37-client") connect_many(client, [source]) - wait_remote_game(client, PERF_GAME_ID, peer_count=1, version="20260520") + wait_remote_game(client, PERF_GAME_ID, peer_count=1, version=PERF_GAME_VERSION) waiter = LineWaiter(len(client.output)) client.send({"cmd": "download", "game_id": PERF_GAME_ID, "install": False}) @@ -1046,7 +1075,7 @@ class Runner: throughput = finished.get("data", {}).get("throughput") if not throughput: raise ScenarioError(f"download-finished did not include throughput: {finished}") - expected_bytes = PERF_GAME_SIZE + len("20260520") + expected_bytes = PERF_GAME_SIZE + len(PERF_GAME_VERSION) if int(throughput["bytes"]) != expected_bytes: raise ScenarioError( f"throughput byte count mismatch: {throughput['bytes']} != {expected_bytes}" @@ -1060,6 +1089,535 @@ class Runner: f"{throughput['chunks']} chunks" ) + def s38_first_play_launch_settings(self) -> str: + client_dir = self.fixture_root / "s38-client" + copy_game("css", client_dir) + client = self.peer( + "s38-client", + games_dir=client_dir, + extra_args=["--unrar", "/usr/local/bin/unrar"], + ) + waiter = LineWaiter(len(client.output)) + client.send({"cmd": "install", "game_id": "css"}) + client.wait_for( + event_is("install-finished", "css"), + timeout=30, + description="css install", + waiter=waiter, + ) + wait_local_game(client, "css", downloaded=True, installed=True) + + marker = client.host_state_dir / "games" / "css" / "launch_settings_applied" + if marker.exists(): + raise ScenarioError("launch settings marker existed before first play") + + local_root = client.host_games_dir / "css" / "local" + account_file = local_root / "profiles" / "local" / "account_name.txt" + language_file = local_root / "profiles" / "local" / "language.txt" + ini_file = ( + local_root + / "engine" + / "bin" + / "win64" + / "steam_settings" + / "SmartSteamEmu.ini" + ) + for path in [account_file, language_file, ini_file]: + if not path.is_file(): + raise ScenarioError(f"expected installed launch settings file: {path}") + if b"PersonaName = stubplayer\r\n" not in ini_file.read_bytes(): + raise ScenarioError("installed SmartSteamEmu.ini did not preserve CRLF stub PersonaName") + + first = client.send( + { + "cmd": "play", + "game_id": "css", + "username": "Lan Hero", + "language": "german", + } + )["data"]["outcome"] + expected_first = { + "already_applied": False, + "account_name_written": True, + "language_written": True, + "persona_name_written": True, + } + if first != expected_first: + raise ScenarioError(f"unexpected first play outcome: {first}") + if not marker.is_file(): + raise ScenarioError("launch settings marker was not written after first play") + if account_file.read_text(encoding="utf-8") != "Lan Hero": + raise ScenarioError("account_name.txt was not stamped with username") + if language_file.read_text(encoding="utf-8") != "german": + raise ScenarioError("language.txt was not stamped with language") + stamped_ini = ini_file.read_bytes() + if b"PersonaName = Lan Hero\r\n" not in stamped_ini: + raise ScenarioError("PersonaName was not stamped with CRLF preserved") + if b"AppId = 240\r\n" not in stamped_ini or b"Language = english\r\n" not in stamped_ini: + raise ScenarioError("SmartSteamEmu.ini sibling lines were not preserved") + + client.docker_exec( + "sh", + "-c", + "printf resetaccount > /games/css/local/profiles/local/account_name.txt", + ) + client.docker_exec( + "sh", + "-c", + "printf resetlang > /games/css/local/profiles/local/language.txt", + ) + client.docker_exec( + "sh", + "-c", + "printf '[Settings]\\r\\nAppId = 240\\r\\n" + "PersonaName = resetplayer\\r\\nLanguage = english\\r\\n' > " + "/games/css/local/engine/bin/win64/steam_settings/SmartSteamEmu.ini", + ) + + second = client.send( + { + "cmd": "play", + "game_id": "css", + "username": "Second User", + "language": "french", + } + )["data"]["outcome"] + expected_second = { + "already_applied": True, + "account_name_written": False, + "language_written": False, + "persona_name_written": False, + } + if second != expected_second: + raise ScenarioError(f"unexpected second play outcome: {second}") + if account_file.read_text(encoding="utf-8") != "resetaccount": + raise ScenarioError("second play rewrote account_name.txt despite marker") + if language_file.read_text(encoding="utf-8") != "resetlang": + raise ScenarioError("second play rewrote language.txt despite marker") + if b"PersonaName = resetplayer\r\n" not in ini_file.read_bytes(): + raise ScenarioError("second play rewrote PersonaName despite marker") + + return "css first play stamped launch settings once; second play respected the marker" + + def stream_install_cnctw(self, prefix: str) -> tuple[Peer, Peer]: + source_dir = self.fixture_root / f"{prefix}-bravo" + copy_game("cnctw", source_dir, version="20160128") + source = self.peer(f"{prefix}-bravo", games_dir=source_dir) + client = self.peer(f"{prefix}-client") + connect_many(client, [source]) + wait_remote_game(client, "cnctw", peer_count=1) + waiter = LineWaiter(len(client.output)) + client.send({"cmd": "stream-install", "game_id": "cnctw"}) + client.wait_for( + event_is("download-begin", "cnctw"), + timeout=20, + description="stream begin cnctw", + waiter=waiter, + ) + client.wait_for( + event_is("download-finished", "cnctw"), + timeout=60, + description="stream finish cnctw", + waiter=waiter, + ) + client.wait_for( + event_is("install-finished", "cnctw"), + timeout=30, + description="stream install cnctw", + waiter=waiter, + ) + return source, client + + def s39_streamed_install_local_only(self) -> str: + source, client = self.stream_install_cnctw("s39") + game = wait_local_game(client, "cnctw", downloaded=False, installed=True) + assert_game_state( + game, + downloaded=False, + installed=True, + availability="LocalOnly", + ) + + game_root = client.host_games_dir / "cnctw" + assert_not_exists(game_root / "version.ini") + assert_not_exists(game_root / "cnctw.eti") + + expected = { + "bin/cnctw-payload.bin": unrar_entry_sha256( + source, "cnctw", "bin/cnctw-payload.bin" + ), + "data/cnctw-assets.dat": unrar_entry_sha256( + source, "cnctw", "data/cnctw-assets.dat" + ), + } + actual = { + rel: sha256_file(game_root / "local" / rel) + for rel in expected + } + if actual != expected: + raise ScenarioError(f"streamed local payload hashes mismatched: {actual} != {expected}") + + streamed_bytes = sum( + int(item.get("data", {}).get("length", 0)) + for item in client.output + if item.get("type") == "event" + and item.get("event") == "download-chunk-finished" + and item.get("data", {}).get("game_id") == "cnctw" + ) + expected_bytes = 3 * 1024 * 1024 + if streamed_bytes != expected_bytes: + raise ScenarioError( + f"streamed byte count mismatch: {streamed_bytes} != {expected_bytes}" + ) + + wait_no_outbound_transfer(source, "cnctw") + + return ( + "cnctw streamed into local/ only; root archive and version.ini absent; " + f"payload hashes={actual}; source outbound transfer drained" + ) + + def s40_streamed_receiver_not_source(self) -> str: + _source, receiver = self.stream_install_cnctw("s40") + observer = self.peer("s40-observer") + connect_many(observer, [receiver]) + receiver_snapshot = wait_peer_has_game(observer, receiver.peer_id, "cnctw") + summary = next( + game + for game in receiver_snapshot.get("games", []) + if game.get("id") == "cnctw" + ) + if summary.get("availability") != "LocalOnly" or summary.get("downloaded"): + raise ScenarioError(f"receiver did not advertise cnctw as local-only: {summary}") + + wait_remote_absent(observer, "cnctw", timeout=5) + err = observer.send( + {"cmd": "download", "game_id": "cnctw", "install": False}, + expect_error=True, + ) + if "no peers have game cnctw" not in err["error"]: + raise ScenarioError(f"unexpected local-only download error: {err}") + assert_not_exists(observer.host_games_dir / "cnctw") + return ( + "observer saw receiver's local-only cnctw snapshot, but remote aggregation hid it " + f"and download errored '{err['error']}'" + ) + + def s41_solid_archive_streamed_install(self) -> str: + source_dir = self.fixture_root / "s41-solid-source" + source_game = source_dir / "cnctw" + shutil.copytree(FIXTURES / "fixture-solid" / "cnctw", source_game) + + source = self.peer("s41-solid-source", games_dir=source_dir) + assert_peer_rar_archive_solid(source, "cnctw") + client = self.peer("s41-solid-client") + connect_many(client, [source]) + wait_remote_game(client, "cnctw", peer_count=1, version="20160128") + + waiter = LineWaiter(len(client.output)) + client.send({"cmd": "stream-install", "game_id": "cnctw"}) + client.wait_for( + event_is("download-finished", "cnctw"), + timeout=60, + description="solid stream finish cnctw", + waiter=waiter, + ) + client.wait_for( + event_is("install-finished", "cnctw"), + timeout=30, + description="solid stream install cnctw", + waiter=waiter, + ) + + game = wait_local_game(client, "cnctw", downloaded=False, installed=True) + assert_game_state( + game, + downloaded=False, + installed=True, + availability="LocalOnly", + ) + game_root = client.host_games_dir / "cnctw" + assert_not_exists(game_root / "version.ini") + assert_not_exists(game_root / "cnctw.eti") + + expected = { + "bin/cnctw-solid-payload.bin": unrar_entry_sha256( + source, "cnctw", "bin/cnctw-solid-payload.bin" + ), + "data/cnctw-solid-assets.dat": unrar_entry_sha256( + source, "cnctw", "data/cnctw-solid-assets.dat" + ), + } + actual = { + rel: sha256_file(game_root / "local" / rel) + for rel in expected + } + if actual != expected: + raise ScenarioError( + f"solid streamed payload hashes mismatched: {actual} != {expected}" + ) + + streamed_bytes = sum( + int(item.get("data", {}).get("length", 0)) + for item in client.output + if item.get("type") == "event" + and item.get("event") == "download-chunk-finished" + and item.get("data", {}).get("game_id") == "cnctw" + ) + expected_bytes = sum((game_root / "local" / rel).stat().st_size for rel in expected) + if streamed_bytes != expected_bytes: + raise ScenarioError( + f"solid streamed byte count mismatch: {streamed_bytes} != {expected_bytes}" + ) + + return ( + "solid cnctw archive streamed through one local-only install; " + f"payload hashes={actual}, bytes={streamed_bytes}" + ) + + def s42_streamed_install_retries_next_source(self) -> str: + bad_dir = self.fixture_root / "s42-bad-source" + good_dir = self.fixture_root / "s42-good-source" + copy_game("cnctw", bad_dir, version="20160128") + copy_game("cnctw", good_dir, version="20160128") + + bad = self.peer( + "s42-bad-source", + games_dir=bad_dir, + extra_args=["--unrar", "/missing-unrar"], + ) + good = self.peer("s42-good-source", games_dir=good_dir) + if socket_addr_sort_key(bad.ready_addr) > socket_addr_sort_key(good.ready_addr): + raise ScenarioError( + "S42 requires the broken source to sort before the good source; " + f"bad={bad.ready_addr}, good={good.ready_addr}" + ) + + client = self.peer("s42-client") + connect_many(client, [bad, good]) + wait_remote_game(client, "cnctw", peer_count=2, version="20160128") + + waiter = LineWaiter(len(client.output)) + client.send({"cmd": "stream-install", "game_id": "cnctw"}) + client.wait_for( + event_is("download-finished", "cnctw"), + timeout=60, + description="retry stream finish cnctw", + waiter=waiter, + ) + client.wait_for( + event_is("install-finished", "cnctw"), + timeout=30, + description="retry stream install cnctw", + waiter=waiter, + ) + + game = wait_local_game(client, "cnctw", downloaded=False, installed=True) + assert_game_state( + game, + downloaded=False, + installed=True, + availability="LocalOnly", + ) + game_root = client.host_games_dir / "cnctw" + assert_not_exists(game_root / ".local.installing") + assert_not_exists(game_root / "version.ini") + assert_not_exists(game_root / "cnctw.eti") + assert_only_chunk_sources(client, "cnctw", {good.ready_addr}) + + expected = { + "bin/cnctw-payload.bin": unrar_entry_sha256( + good, "cnctw", "bin/cnctw-payload.bin" + ), + "data/cnctw-assets.dat": unrar_entry_sha256( + good, "cnctw", "data/cnctw-assets.dat" + ), + } + actual = { + rel: sha256_file(game_root / "local" / rel) + for rel in expected + } + if actual != expected: + raise ScenarioError(f"retry streamed payload hashes mismatched: {actual} != {expected}") + + streamed_bytes = sum( + int(item.get("data", {}).get("length", 0)) + for item in client.output + if item.get("type") == "event" + and item.get("event") == "download-chunk-finished" + and item.get("data", {}).get("game_id") == "cnctw" + ) + expected_bytes = 3 * 1024 * 1024 + if streamed_bytes != expected_bytes: + raise ScenarioError( + f"retry streamed byte count mismatch: {streamed_bytes} != {expected_bytes}" + ) + + return ( + "broken first source failed without chunks, next source completed whole stream; " + f"good={good.ready_addr}, bad={bad.ready_addr}, bytes={streamed_bytes}" + ) + + def s43_streamed_install_rejects_installed_game(self) -> str: + _source, client = self.stream_install_cnctw("s43") + + start = len(client.output) + waiter = LineWaiter(start) + client.send({"cmd": "stream-install", "game_id": "cnctw"}) + client.wait_for( + event_is("download-failed", "cnctw"), + timeout=20, + description="already-installed stream rejection", + waiter=waiter, + ) + assert_no_event_since(client, start, "install-finished", "cnctw") + assert_no_event_since(client, start, "download-finished", "cnctw") + wait_no_active(client, "cnctw") + + game = wait_local_game(client, "cnctw", downloaded=False, installed=True) + assert_game_state( + game, + downloaded=False, + installed=True, + availability="LocalOnly", + ) + return "already-installed cnctw rejected a second streamed install without state drift" + + def s44_corrupt_stream_rolls_back(self) -> str: + source_dir = self.fixture_root / "s44-corrupt-source" + copy_game("cnctw", source_dir, version="20160128") + (source_dir / "cnctw" / "cnctw.eti").write_bytes(b"not a rar archive") + + source = self.peer("s44-corrupt-source", games_dir=source_dir) + client = self.peer("s44-client") + connect_many(client, [source]) + wait_remote_game(client, "cnctw", peer_count=1, version="20160128") + + start = len(client.output) + waiter = LineWaiter(start) + client.send({"cmd": "stream-install", "game_id": "cnctw"}) + client.wait_for( + event_is("download-failed", "cnctw"), + timeout=30, + description="corrupt stream failed", + waiter=waiter, + ) + assert_no_event_since(client, start, "download-finished", "cnctw") + assert_no_event_since(client, start, "install-finished", "cnctw") + wait_no_active(client, "cnctw") + assert_failed_stream_left_no_local(client, "cnctw") + return "corrupt cnctw archive emitted download-failed and left no local install" + + def s45_sender_disconnect_mid_stream(self) -> str: + source_dir = self.fixture_root / "s45-source" + copy_game("alienswarm", source_dir, version="20190317") + source = self.peer("s45-source", games_dir=source_dir) + client = self.peer("s45-client") + connect_many(client, [source]) + wait_remote_game(client, "alienswarm", peer_count=1, version="20190317") + + start = len(client.output) + waiter = LineWaiter(start) + client.send({"cmd": "stream-install", "game_id": "alienswarm"}) + client.wait_for( + event_is("download-chunk-finished", "alienswarm"), + timeout=30, + description="first alienswarm stream chunk before source drop", + waiter=waiter, + ) + source.kill() + terminal = client.wait_for( + event_name_in({"download-failed", "download-peers-gone"}, "alienswarm"), + timeout=60, + description="sender disconnect terminal event", + waiter=waiter, + ) + assert_no_event_since(client, start, "download-finished", "alienswarm") + assert_no_event_since(client, start, "install-finished", "alienswarm") + wait_no_active(client, "alienswarm") + assert_failed_stream_left_no_local(client, "alienswarm") + return ( + "sender disconnect after first alienswarm chunk rolled back stream; " + f"terminal={terminal['event']}" + ) + + def s46_receiver_cancel_mid_stream(self) -> str: + source_dir = self.fixture_root / "s46-source" + copy_game("alienswarm", source_dir, version="20190317") + source = self.peer("s46-source", games_dir=source_dir) + client = self.peer("s46-client") + connect_many(client, [source]) + wait_remote_game(client, "alienswarm", peer_count=1, version="20190317") + + start = len(client.output) + waiter = LineWaiter(start) + client.send({"cmd": "stream-install", "game_id": "alienswarm"}) + client.wait_for( + event_is("download-chunk-finished", "alienswarm"), + timeout=30, + description="first alienswarm stream chunk before receiver cancel", + waiter=waiter, + ) + client.send({"cmd": "cancel-download", "game_id": "alienswarm"}) + wait_no_active(client, "alienswarm", timeout=60) + assert_no_event_since(client, start, "download-finished", "alienswarm") + assert_no_event_since(client, start, "download-failed", "alienswarm") + assert_no_event_since(client, start, "install-finished", "alienswarm") + assert_failed_stream_left_no_local(client, "alienswarm") + return "receiver cancel after first alienswarm chunk rolled back without failed event" + + def s47_multi_archive_streams_in_sorted_order(self) -> str: + source_dir = self.fixture_root / "s47-source" + source_game = source_dir / "cnctw" + shutil.copytree(FIXTURES / "fixture-multi" / "cnctw", source_game) + + source = self.peer("s47-source", games_dir=source_dir) + client = self.peer("s47-client") + connect_many(client, [source]) + wait_remote_game(client, "cnctw", peer_count=1, version="20160128") + + waiter = LineWaiter(len(client.output)) + client.send({"cmd": "stream-install", "game_id": "cnctw"}) + client.wait_for( + event_is("download-finished", "cnctw"), + timeout=30, + description="multi-archive stream finish", + waiter=waiter, + ) + client.wait_for( + event_is("install-finished", "cnctw"), + timeout=30, + description="multi-archive stream install", + waiter=waiter, + ) + + game = wait_local_game(client, "cnctw", downloaded=False, installed=True) + assert_game_state( + game, + downloaded=False, + installed=True, + availability="LocalOnly", + ) + game_root = client.host_games_dir / "cnctw" + assert_not_exists(game_root / "version.ini") + assert_not_exists(game_root / "a-first.eti") + assert_not_exists(game_root / "z-second.eti") + + chunk_paths = streamed_chunk_paths(client, "cnctw") + expected_paths = [ + "cnctw/.local.installing/order/first.txt", + "cnctw/.local.installing/order/second.txt", + ] + if chunk_paths != expected_paths: + raise ScenarioError(f"multi-archive stream order mismatch: {chunk_paths}") + + first = (game_root / "local" / "order" / "first.txt").read_text(encoding="utf-8") + second = (game_root / "local" / "order" / "second.txt").read_text(encoding="utf-8") + if first != "first archive payload\n" or second != "second archive payload\n": + raise ScenarioError(f"multi-archive payload mismatch: {first!r}, {second!r}") + + return f"multi-archive cnctw streamed in sorted order: {chunk_paths}" + def run(command: list[str], description: str) -> subprocess.CompletedProcess[str]: result = subprocess.run( @@ -1128,6 +1686,7 @@ def copy_game(game_id: str, destination_games_dir: Path, *, version: str | None shutil.rmtree(destination) destination.parent.mkdir(parents=True, exist_ok=True) shutil.copytree(source, destination) + version = version if version is not None else CATALOG_VERSIONS.get(game_id) if version is not None: (destination / "version.ini").write_text(version, encoding="utf-8") @@ -1164,19 +1723,62 @@ def create_many_small_game(root: Path) -> None: for index in range(20): child = root / f"file-{index:02}.bin" child.write_bytes(hashlib.sha256(f"small-{index}".encode()).digest() * 8) - (root / "version.ini").write_text("20250101", encoding="utf-8") + (root / "version.ini").write_text(CATALOG_VERSIONS.get(root.name, "20250101"), encoding="utf-8") def create_large_sparse_game(root: Path, *, size: int) -> None: if root.exists(): shutil.rmtree(root) root.mkdir(parents=True) - (root / "version.ini").write_text("20260520", encoding="utf-8") + (root / "version.ini").write_text(PERF_GAME_VERSION, encoding="utf-8") archive = root / f"{root.name}.eti" with archive.open("wb") as handle: handle.truncate(size) +def sha256_file(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def unrar_entry_sha256(peer: Peer, game_id: str, relative_path: str) -> str: + command = ( + f"unrar p -inul /games/{shlex.quote(game_id)}/{shlex.quote(game_id)}.eti " + f"{shlex.quote(relative_path)} | sha256sum" + ) + output = peer.docker_exec("sh", "-c", command).stdout.strip() + if not output: + raise ScenarioError(f"empty sha256 output for {game_id}:{relative_path}") + return output.split()[0] + + +def assert_peer_rar_archive_solid(peer: Peer, game_id: str) -> None: + output = peer.docker_exec( + "unrar", + "lt", + "-cfg-", + f"/games/{game_id}/{game_id}.eti", + ).stdout + for line in output.splitlines(): + stripped = line.strip() + if stripped.startswith("Details:"): + if "solid" in stripped.lower(): + return + raise ScenarioError(f"RAR archive is not solid: {game_id}") + raise ScenarioError(f"RAR archive details were not reported: {game_id}") + + +def socket_addr_sort_key(addr: str | None) -> tuple[int, int]: + if addr is None: + raise ScenarioError("cannot sort missing peer address") + host, port = addr.rsplit(":", 1) + host = host.removeprefix("[").removesuffix("]") + return (int(ipaddress.ip_address(host)), int(port)) + + def format_bytes(size: int) -> str: return f"{size / 1024 / 1024 / 1024:.2f} GiB" @@ -1253,6 +1855,32 @@ def wait_local_game( ) +def wait_no_active(peer: Peer, game_id: str, timeout: float = 20) -> None: + deadline = time.monotonic() + timeout + last_active: list[dict[str, Any]] = [] + while time.monotonic() < deadline: + active = peer.status()["active_operations"] + last_active = active + if all(item["game_id"] != game_id for item in active): + return + time.sleep(0.4) + raise ScenarioError(f"{peer.name} still has active operation for {game_id}: {last_active}") + + +def wait_no_outbound_transfer(peer: Peer, game_id: str, timeout: float = 20) -> None: + deadline = time.monotonic() + timeout + last_active: dict[str, int] = {} + while time.monotonic() < deadline: + active = peer.status()["active_outbound_transfers"] + last_active = active + if active.get(game_id, 0) == 0: + return + time.sleep(0.4) + raise ScenarioError( + f"{peer.name} still has outbound transfer for {game_id}: {last_active}" + ) + + def assert_game_state( game: dict[str, Any], *, @@ -1299,7 +1927,10 @@ def wait_peer_has_game( def assert_local_absent(peer: Peer, game_id: str) -> None: rows = peer.list_games()["local"] - if any(row["id"] == game_id and row.get("downloaded") for row in rows): + if any( + row["id"] == game_id and (row.get("downloaded") or row.get("installed")) + for row in rows + ): raise ScenarioError(f"{peer.name} advertises failed local {game_id}: {rows}") @@ -1315,6 +1946,15 @@ def assert_not_exists(path: Path) -> None: raise ScenarioError(f"expected path to be absent: {path}") +def assert_failed_stream_left_no_local(peer: Peer, game_id: str) -> None: + game_root = peer.host_games_dir / game_id + assert_local_absent(peer, game_id) + assert_not_exists(game_root / "local") + assert_not_exists(game_root / ".local.installing") + assert_not_exists(game_root / "version.ini") + assert_not_exists(game_root / f"{game_id}.eti") + + def event_is(event: str, game_id: str | None = None) -> Callable[[dict[str, Any]], bool]: def predicate(item: dict[str, Any]) -> bool: if item.get("type") != "event" or item.get("event") != event: @@ -1326,6 +1966,17 @@ def event_is(event: str, game_id: str | None = None) -> Callable[[dict[str, Any] return predicate +def event_name_in(events: set[str], game_id: str | None = None) -> Callable[[dict[str, Any]], bool]: + def predicate(item: dict[str, Any]) -> bool: + if item.get("type") != "event" or item.get("event") not in events: + return False + if game_id is None: + return True + return item.get("data", {}).get("game_id") == game_id + + return predicate + + def assert_no_event(peer: Peer, waiter: LineWaiter, event: str, game_id: str) -> None: for item in peer.output[waiter.seen :]: if item.get("type") == "event" and item.get("event") == event: @@ -1333,6 +1984,13 @@ def assert_no_event(peer: Peer, waiter: LineWaiter, event: str, game_id: str) -> raise ScenarioError(f"unexpected {event} for {game_id}: {item}") +def assert_no_event_since(peer: Peer, start: int, event: str, game_id: str) -> None: + for item in peer.output[start:]: + if item.get("type") == "event" and item.get("event") == event: + if item.get("data", {}).get("game_id") == game_id: + raise ScenarioError(f"unexpected {event} for {game_id}: {item}") + + def assert_only_chunk_sources( peer: Peer, game_id: str, @@ -1358,6 +2016,16 @@ def assert_only_chunk_sources( raise ScenarioError(f"no chunk events recorded for {game_id}") +def streamed_chunk_paths(peer: Peer, game_id: str) -> list[str]: + return [ + item["data"]["relative_path"] + for item in peer.output + if item.get("type") == "event" + and item.get("event") == "download-chunk-finished" + and item.get("data", {}).get("game_id") == game_id + ] + + def chunk_totals(peer: Peer, game_id: str, relative_path: str) -> dict[str, int]: totals: dict[str, int] = {} for item in peer.output: diff --git a/crates/lanspread-peer-cli/src/lib.rs b/crates/lanspread-peer-cli/src/lib.rs index 78e4315..7ed9f35 100644 --- a/crates/lanspread-peer-cli/src/lib.rs +++ b/crates/lanspread-peer-cli/src/lib.rs @@ -33,6 +33,12 @@ pub enum CliCommand { game_id: String, install_after_download: bool, }, + StreamInstall { + game_id: String, + }, + CancelDownload { + game_id: String, + }, Install { game_id: String, }, @@ -63,6 +69,8 @@ impl CliCommand { Self::ListGames => "list-games", Self::SetGameDir { .. } => "set-game-dir", Self::Download { .. } => "download", + Self::StreamInstall { .. } => "stream-install", + Self::CancelDownload { .. } => "cancel-download", Self::Install { .. } => "install", Self::Uninstall { .. } => "uninstall", Self::Play { .. } => "play", @@ -101,6 +109,12 @@ pub fn parse_command_value(value: &Value) -> eyre::Result { game_id: game_id(object)?, install_after_download: install_after_download(object)?, }, + "stream-install" => CliCommand::StreamInstall { + game_id: game_id(object)?, + }, + "cancel-download" => CliCommand::CancelDownload { + game_id: game_id(object)?, + }, "install" => CliCommand::Install { game_id: game_id(object)?, }, @@ -344,6 +358,32 @@ mod tests { assert_eq!(parsed["data"]["peer_count"], 0); } + #[test] + fn parses_stream_install_command() { + let parsed = parse_command_line(r#"{"cmd":"stream-install","game_id":"cnctw"}"#) + .expect("command should parse"); + + assert_eq!( + parsed.command, + CliCommand::StreamInstall { + game_id: "cnctw".to_string(), + } + ); + } + + #[test] + fn parses_cancel_download_command() { + let parsed = parse_command_line(r#"{"cmd":"cancel-download","game_id":"cnctw"}"#) + .expect("command should parse"); + + assert_eq!( + parsed.command, + CliCommand::CancelDownload { + game_id: "cnctw".to_string(), + } + ); + } + #[tokio::test] async fn fixture_unpacker_creates_install_payload() { let temp = TempDir::new("lanspread-peer-cli-fixture"); diff --git a/crates/lanspread-peer-cli/src/main.rs b/crates/lanspread-peer-cli/src/main.rs index f5155fa..21ff267 100644 --- a/crates/lanspread-peer-cli/src/main.rs +++ b/crates/lanspread-peer-cli/src/main.rs @@ -16,7 +16,10 @@ use lanspread_db::db::{Game, GameCatalog, GameFileDescription}; use lanspread_peer::{ ActiveOperation, ActiveOperationKind, + ExternalUnrarStreamProvider, InstallOperation, + NoopStreamInstallProvider, + OutboundTransfers, PeerCommand, PeerEvent, PeerGameDB, @@ -24,6 +27,7 @@ use lanspread_peer::{ PeerRuntimeHandle, PeerSnapshot, PeerStartOptions, + StreamInstallProvider, migrate_legacy_state, start_peer_with_options, }; @@ -116,6 +120,7 @@ struct SharedState { state: RwLock, peer_game_db: Arc>, catalog: Arc>, + active_outbound_transfers: OutboundTransfers, notify: Notify, games_dir: PathBuf, state_dir: PathBuf, @@ -134,10 +139,16 @@ async fn main() -> eyre::Result<()> { let (tx_events, rx_events) = mpsc::unbounded_channel(); let peer_game_db = Arc::new(RwLock::new(PeerGameDB::new())); let catalog = Arc::new(RwLock::new(catalog)); - let unpacker: Arc = match args.unrar { + let active_outbound_transfers: OutboundTransfers = Arc::new(RwLock::new(HashMap::new())); + let unrar_for_streaming = args.unrar.clone().or_else(default_unrar_program); + let unpacker: Arc = match args.unrar.clone() { Some(path) => Arc::new(ExternalUnrarUnpacker::new(path)), None => Arc::new(FixtureUnpacker), }; + let stream_install_provider: Arc = match unrar_for_streaming { + Some(path) => Arc::new(ExternalUnrarStreamProvider::new(path)), + None => Arc::new(NoopStreamInstallProvider), + }; let mut handle = start_peer_with_options( args.games_dir.clone(), @@ -147,7 +158,8 @@ async fn main() -> eyre::Result<()> { catalog.clone(), PeerStartOptions { state_dir: Some(args.state_dir.clone()), - active_outbound_transfers: None, + active_outbound_transfers: Some(active_outbound_transfers.clone()), + stream_install_provider: Some(stream_install_provider), }, )?; let sender = handle.sender(); @@ -156,6 +168,7 @@ async fn main() -> eyre::Result<()> { state: RwLock::new(CliState::default()), peer_game_db, catalog: catalog.clone(), + active_outbound_transfers, notify: Notify::new(), games_dir: args.games_dir.clone(), state_dir: args.state_dir.clone(), @@ -249,6 +262,21 @@ async fn handle_command( })?; Ok(json!({"queued": true, "game_id": game_id, "install": install_after_download})) } + CliCommand::StreamInstall { game_id } => { + ensure_catalog_game(shared, game_id).await?; + ensure_no_active_operation(shared, game_id).await?; + sender.send(PeerCommand::StreamInstallGame { + id: game_id.clone(), + })?; + Ok(json!({"queued": true, "game_id": game_id})) + } + CliCommand::CancelDownload { game_id } => { + ensure_catalog_game(shared, game_id).await?; + sender.send(PeerCommand::CancelDownload { + id: game_id.clone(), + })?; + Ok(json!({"queued": true, "game_id": game_id})) + } CliCommand::Install { game_id } => { ensure_catalog_game(shared, game_id).await?; ensure_no_active_operation(shared, game_id).await?; @@ -289,12 +317,20 @@ async fn handle_command( async fn status(shared: &SharedState) -> eyre::Result { let state = shared.state.read().await; let peer_count = shared.peer_game_db.read().await.peer_snapshots().len(); + let active_outbound_transfers = { + let active = shared.active_outbound_transfers.read().await; + active + .iter() + .map(|(game_id, transfers)| (game_id.clone(), transfers.len())) + .collect::>() + }; Ok(json!({ "local_peer": state.local_peer.clone(), "peer_count": peer_count, "local_games": state.local_games.len(), "remote_games": state.remote_games.len(), "active_operations": active_operations_json(&state.active_operations), + "active_outbound_transfers": active_outbound_transfers, })) } @@ -729,6 +765,15 @@ fn default_catalog_db() -> Option { .find(|path| path.exists()) } +fn default_unrar_program() -> Option { + [ + PathBuf::from("/usr/local/bin/unrar"), + PathBuf::from("/usr/bin/unrar"), + ] + .into_iter() + .find(|path| path.exists()) +} + fn next_string(args: &mut impl Iterator, flag: &str) -> eyre::Result { args.next() .ok_or_else(|| eyre::eyre!("{flag} requires a value"))? diff --git a/crates/lanspread-peer/ARCHITECTURE.md b/crates/lanspread-peer/ARCHITECTURE.md index 0f8c080..a42e6e0 100644 --- a/crates/lanspread-peer/ARCHITECTURE.md +++ b/crates/lanspread-peer/ARCHITECTURE.md @@ -166,6 +166,18 @@ Most scans become O(number of game dirs), with full recursion only when needed. scratch sentinel files. `local/` and install transaction metadata are preserved, so a cancelled update of an installed game settles as local-only. +### Streamed install integrity + +- Low-disk streamed installs request archive-derived file bytes from one peer + and write them directly into the install transaction staging directory. +- The receiver verifies every streamed file against the sender archive's file + size and RAR CRC32 before the transaction may commit. This catches truncated + streams, transport corruption, and provider bugs. +- This is not malicious-peer protection: the peer controls both the archive + metadata and the streamed bytes. A trusted-content model needs catalog-owned + hashes, either for the root archives or for extracted files, and receiver-side + SHA-256 verification against those catalog values before commit. + ## Fault tolerance rules - Every peer is keyed by `peer_id`, not by IP address. diff --git a/crates/lanspread-peer/Cargo.toml b/crates/lanspread-peer/Cargo.toml index e580140..6437e69 100644 --- a/crates/lanspread-peer/Cargo.toml +++ b/crates/lanspread-peer/Cargo.toml @@ -14,6 +14,7 @@ lanspread-utils = { path = "../lanspread-utils" } # external bytes = { workspace = true } +crc32fast = { workspace = true } eyre = { workspace = true } futures = { workspace = true } gethostname = { workspace = true } diff --git a/crates/lanspread-peer/README.md b/crates/lanspread-peer/README.md index 4e34ef8..55a409b 100644 --- a/crates/lanspread-peer/README.md +++ b/crates/lanspread-peer/README.md @@ -14,8 +14,8 @@ It is designed to run headless – other crates (most notably roots are announced or served. - `PeerCommand` represents the small control surface exposed to the UI layer: `ListGames`, `GetGame`, `FetchLatestFromPeers`, `DownloadGameFiles`, - `InstallGame`, `UninstallGame`, `RemoveDownloadedGame`, `CancelDownload`, - `SetGameDir`, and `GetPeerCount`. + `StreamInstallGame`, `InstallGame`, `UninstallGame`, `RemoveDownloadedGame`, + `CancelDownload`, `SetGameDir`, and `GetPeerCount`. - `PeerEvent` enumerates everything the peer runtime reports back to the UI: library snapshots, download/install/uninstall lifecycle updates, runtime failures, and peer membership changes. @@ -28,8 +28,8 @@ lifetime of the process: 1. **Server component** (`run_server_component`) – listens for QUIC connections, advertises via mDNS, and serves `Request::ListGames`, `Request::GetGame`, - `Request::GetGameFileData`, and `Request::GetGameFileChunk` by reading from - the local game directory. + `Request::GetGameFileData`, `Request::GetGameFileChunk`, and + `Request::StreamInstall` by reading from the local game directory. 2. **Discovery loop** (`run_peer_discovery`) – uses the `lanspread-mdns` helper to discover other peers. The blocking mDNS work is executed on a dedicated thread via `tokio::task::spawn_blocking` so that the Tokio runtime @@ -87,6 +87,26 @@ When the UI asks to download a game: 7. After a successful sentinel commit, `PeerEvent::DownloadGameFilesFinished` is emitted and the peer auto-runs the install transaction. +### Streamed Install Pipeline + +Low-disk installs use `PeerCommand::StreamInstallGame` instead of the normal +archive download pipeline. The peer core owns the whole operation: it refreshes +file metadata from catalog-version peers, runs the same majority file-size +validation used by normal downloads, selects a validated peer list, and emits +the regular download/install lifecycle events while streaming archive-expanded +bytes directly into a `StreamedInstallTransaction`. + +The sender-side `StreamInstallProvider` writes control and chunk frames through +a cancellable `StreamInstallFrameSink`. If the QUIC writer fails because the +receiver cancelled or disconnected, the sink wakes any producer blocked on the +bounded frame channel and lets the transfer guard drop normally. + +Each failed peer attempt rolls back its staging directory before trying the next +validated peer. A transaction that created a previously missing game root +removes that root again when rollback leaves it empty. Once staging has been +renamed to `local/`, post-promote intent or launch-settings cleanup failures are +logged for startup recovery rather than reported as a failed install. + `PeerCommand::CancelDownload` cancels the tracked download token for an active transfer. The transfer task remains responsible for clearing `active_operations`, discarding partial payload files, and refreshing the settled local snapshot, so diff --git a/crates/lanspread-peer/src/context.rs b/crates/lanspread-peer/src/context.rs index a4f3da3..db47b93 100644 --- a/crates/lanspread-peer/src/context.rs +++ b/crates/lanspread-peer/src/context.rs @@ -6,7 +6,14 @@ use lanspread_db::db::{GameCatalog, GameDB}; use tokio::sync::{RwLock, mpsc::UnboundedSender}; use tokio_util::{sync::CancellationToken, task::TaskTracker}; -use crate::{PeerEvent, Unpacker, events, library::LocalLibraryState, peer_db::PeerGameDB}; +use crate::{ + PeerEvent, + StreamInstallProvider, + Unpacker, + events, + library::LocalLibraryState, + peer_db::PeerGameDB, +}; /// Thread-safe map of active outbound file transfers grouped by game ID. pub type OutboundTransfers = Arc>>>; @@ -38,6 +45,7 @@ pub struct Ctx { pub active_operations: Arc>>, pub active_downloads: Arc>>, pub unpacker: Arc, + pub stream_install_provider: Arc, pub catalog: Arc>, pub peer_id: Arc, pub shutdown: CancellationToken, @@ -57,6 +65,7 @@ pub struct PeerCtx { pub catalog: Arc>, pub peer_id: Arc, pub tx_notify_ui: tokio::sync::mpsc::UnboundedSender, + pub stream_install_provider: Arc, pub shutdown: CancellationToken, pub task_tracker: TaskTracker, pub active_outbound_transfers: OutboundTransfers, @@ -86,6 +95,7 @@ impl Ctx { task_tracker: TaskTracker, catalog: Arc>, active_outbound_transfers: OutboundTransfers, + stream_install_provider: Arc, ) -> Self { Self { game_dir: Arc::new(RwLock::new(game_dir)), @@ -97,6 +107,7 @@ impl Ctx { active_operations: Arc::new(RwLock::new(HashMap::new())), active_downloads: Arc::new(RwLock::new(HashMap::new())), unpacker, + stream_install_provider, catalog, peer_id: Arc::new(peer_id), shutdown, @@ -120,6 +131,7 @@ impl Ctx { catalog: self.catalog.clone(), peer_id: self.peer_id.clone(), tx_notify_ui, + stream_install_provider: self.stream_install_provider.clone(), shutdown: self.shutdown.clone(), task_tracker: self.task_tracker.clone(), active_outbound_transfers: self.active_outbound_transfers.clone(), diff --git a/crates/lanspread-peer/src/handlers.rs b/crates/lanspread-peer/src/handlers.rs index 9fe7f91..10e68c9 100644 --- a/crates/lanspread-peer/src/handlers.rs +++ b/crates/lanspread-peer/src/handlers.rs @@ -11,6 +11,7 @@ use std::{ use lanspread_db::db::{GameDB, GameFileDescription}; use tokio::sync::{RwLock, mpsc::UnboundedSender}; +use tokio_util::sync::CancellationToken; use crate::{ InstallOperation, @@ -33,6 +34,7 @@ use crate::{ peer_db::PeerGameDB, remote_peer::ensure_peer_id_for_addr, services::{HandshakeCtx, perform_handshake_with_peer}, + stream_install::receive_streamed_install, }; // ============================================================================= @@ -450,6 +452,60 @@ pub async fn handle_install_game_command( spawn_install_operation(ctx, tx_notify_ui, id); } +pub async fn handle_stream_install_game_command( + ctx: &Ctx, + tx_notify_ui: &UnboundedSender, + id: String, +) { + if !catalog_contains(ctx, &id).await { + log::warn!("Ignoring streamed install command for non-catalog game {id}"); + send_download_failed(tx_notify_ui, &id); + return; + } + + let games_folder = { ctx.game_dir.read().await.clone() }; + let game_root = games_folder.join(&id); + if local_dir_is_directory(&game_root).await { + log::warn!("Ignoring streamed install command for already-installed game {id}"); + send_download_failed(tx_notify_ui, &id); + return; + } + + match begin_operation(ctx, tx_notify_ui, &id, OperationKind::Downloading).await { + BeginOperationResult::Started => {} + BeginOperationResult::AlreadyActive => { + log::warn!("Operation for {id} already in progress; ignoring streamed install request"); + return; + } + BeginOperationResult::DrainTimedOut => { + log::error!("Timed out waiting for outbound transfers before streamed install of {id}"); + send_download_failed(tx_notify_ui, &id); + return; + } + } + + let expected_version = catalog_expected_version(ctx, &id).await; + let cancel_token = ctx.shutdown.child_token(); + ctx.active_downloads + .write() + .await + .insert(id.clone(), cancel_token.clone()); + + let ctx_clone = ctx.clone(); + let tx_notify_ui = tx_notify_ui.clone(); + ctx.task_tracker.spawn(async move { + run_stream_install_operation( + ctx_clone, + tx_notify_ui, + id, + game_root, + expected_version, + cancel_token, + ) + .await; + }); +} + /// Handles the `UninstallGame` command. pub async fn handle_uninstall_game_command( ctx: &Ctx, @@ -490,6 +546,284 @@ pub async fn handle_cancel_download_command( cancel_token.cancel(); } +async fn run_stream_install_operation( + ctx: Ctx, + tx_notify_ui: UnboundedSender, + id: String, + game_root: PathBuf, + expected_version: Option, + cancel_token: CancellationToken, +) { + let download_guard = OperationGuard::download( + id.clone(), + ctx.active_operations.clone(), + ctx.active_downloads.clone(), + tx_notify_ui.clone(), + ); + + events::send( + &tx_notify_ui, + PeerEvent::DownloadGameFilesBegin { id: id.clone() }, + ); + + let peer_addrs = + match select_stream_install_peers(&ctx, &id, expected_version.as_deref(), &cancel_token) + .await + { + Ok(peers) => peers, + Err(err) => { + let download_was_cancelled = cancel_token.is_cancelled(); + if download_was_cancelled { + log::info!("Streamed install preflight cancelled for {id}: {err}"); + } else { + log::error!("Streamed install preflight failed for {id}: {err}"); + } + finish_failed_stream_download( + &ctx, + &tx_notify_ui, + &id, + download_guard, + download_was_cancelled, + ) + .await; + return; + } + }; + + match receive_streamed_install_from_peers( + &ctx, + &tx_notify_ui, + &id, + &game_root, + &peer_addrs, + &cancel_token, + ) + .await + { + Ok(transaction) => { + if transition_download_to_install(&ctx, &tx_notify_ui, &id, OperationKind::Installing) + .await + { + clear_active_download(&ctx, &id).await; + send_download_finished(&tx_notify_ui, &id); + download_guard.disarm(); + commit_streamed_install(&ctx, &tx_notify_ui, id, transaction).await; + return; + } + + if let Err(err) = transaction.rollback().await { + log::error!("Failed to roll back streamed install for {id}: {err}"); + } + finish_failed_stream_download(&ctx, &tx_notify_ui, &id, download_guard, false).await; + } + Err(err) => { + let download_was_cancelled = cancel_token.is_cancelled(); + if download_was_cancelled { + log::info!("Streamed install download cancelled for {id}: {err}"); + } else { + log::error!("Streamed install download failed for {id}: {err}"); + } + finish_failed_stream_download( + &ctx, + &tx_notify_ui, + &id, + download_guard, + download_was_cancelled, + ) + .await; + } + } +} + +async fn receive_streamed_install_from_peers( + ctx: &Ctx, + tx_notify_ui: &UnboundedSender, + id: &str, + game_root: &Path, + peer_addrs: &[SocketAddr], + cancel_token: &CancellationToken, +) -> eyre::Result { + let mut last_receive_error = None; + for &peer_addr in peer_addrs { + if cancel_token.is_cancelled() { + eyre::bail!("streamed install for {id} was cancelled"); + } + + let transaction = + install::begin_streamed_install(game_root, ctx.state_dir.as_ref(), id).await?; + let receive_result = receive_streamed_install( + peer_addr, + id, + transaction.staging_dir(), + tx_notify_ui.clone(), + cancel_token.clone(), + ) + .await; + + match receive_result { + Ok(()) => return Ok(transaction), + Err(err) => { + if let Err(rollback_err) = transaction.rollback().await { + log::error!("Failed to roll back streamed install for {id}: {rollback_err}"); + } + if cancel_token.is_cancelled() { + return Err(err); + } + + log::warn!( + "Streamed install attempt from {peer_addr} failed for {id}; trying another peer if available: {err}" + ); + last_receive_error = Some(err); + } + } + } + + Err(last_receive_error.unwrap_or_else(|| { + eyre::eyre!("streamed install download failed for {id}: no peer attempts were made") + })) +} + +async fn select_stream_install_peers( + ctx: &Ctx, + id: &str, + expected_version: Option<&str>, + cancel_token: &CancellationToken, +) -> eyre::Result> { + let mut metadata_peers = { + ctx.peer_game_db + .read() + .await + .peers_with_expected_version(id, expected_version) + }; + metadata_peers.sort(); + if metadata_peers.is_empty() { + eyre::bail!("no peers have game {id}"); + } + + refresh_stream_install_file_details(ctx, id, &metadata_peers, cancel_token).await?; + + let mut peers = match ctx + .peer_game_db + .read() + .await + .validate_file_sizes_majority(id, expected_version) + { + Ok((validated_files, peer_whitelist, _)) if !validated_files.is_empty() => peer_whitelist, + Ok(_) => { + eyre::bail!("no trusted peers available for streamed install of {id}"); + } + Err(err) => { + return Err(err.wrap_err(format!( + "file size majority validation failed for streamed install {id}" + ))); + } + }; + peers.sort(); + if peers.is_empty() { + eyre::bail!("no peer selected for streamed install of {id}"); + } + + Ok(peers) +} + +async fn refresh_stream_install_file_details( + ctx: &Ctx, + id: &str, + peers: &[SocketAddr], + cancel_token: &CancellationToken, +) -> eyre::Result<()> { + let mut fetched_any = false; + for &peer_addr in peers { + if cancel_token.is_cancelled() { + eyre::bail!("streamed install for {id} was cancelled"); + } + + match request_game_details_and_update(peer_addr, id, ctx.peer_game_db.clone()).await { + Ok(_) => { + log::info!("Fetched streamed-install file list for {id} from peer {peer_addr}"); + fetched_any = true; + } + Err(err) => { + log::error!( + "Failed to fetch streamed-install files for {id} from {peer_addr}: {err}" + ); + } + } + } + + if !fetched_any { + eyre::bail!("failed to retrieve game files for {id} from any peer"); + } + + Ok(()) +} + +async fn finish_failed_stream_download( + ctx: &Ctx, + tx_notify_ui: &UnboundedSender, + id: &str, + guard: OperationGuard, + cancelled: bool, +) { + if let Err(err) = refresh_local_game_for_ending_operation(ctx, tx_notify_ui, id).await { + log::error!("Failed to refresh local library after streamed install failure: {err}"); + } + end_download_operation(ctx, tx_notify_ui, id).await; + guard.disarm(); + send_download_failed_unless_cancelled(tx_notify_ui, id, cancelled); +} + +async fn commit_streamed_install( + ctx: &Ctx, + tx_notify_ui: &UnboundedSender, + id: String, + transaction: install::StreamedInstallTransaction, +) { + let operation_guard = OperationGuard::new( + id.clone(), + ctx.active_operations.clone(), + tx_notify_ui.clone(), + ); + events::send( + tx_notify_ui, + PeerEvent::InstallGameBegin { + id: id.clone(), + operation: InstallOperation::Installing, + }, + ); + + match transaction.commit().await { + Ok(()) => { + if let Err(err) = refresh_local_game_for_ending_operation(ctx, tx_notify_ui, &id).await + { + log::error!("Failed to refresh local library after streamed install: {err}"); + } + end_operation(ctx, tx_notify_ui, &id).await; + operation_guard.disarm(); + events::send( + tx_notify_ui, + PeerEvent::InstallGameFinished { id: id.clone() }, + ); + } + Err(err) => { + log::error!("Streamed install commit failed for {id}: {err}"); + if let Err(refresh_err) = + refresh_local_game_for_ending_operation(ctx, tx_notify_ui, &id).await + { + log::error!( + "Failed to refresh local library after streamed install commit failure: {refresh_err}" + ); + } + end_operation(ctx, tx_notify_ui, &id).await; + operation_guard.disarm(); + events::send( + tx_notify_ui, + PeerEvent::InstallGameFailed { id: id.clone() }, + ); + } + } +} + fn spawn_install_operation(ctx: &Ctx, tx_notify_ui: &UnboundedSender, id: String) { let ctx = ctx.clone(); let tx_notify_ui = tx_notify_ui.clone(); @@ -1264,6 +1598,7 @@ mod tests { TaskTracker::new(), Arc::new(RwLock::new(GameCatalog::from_ids(["game".to_string()]))), Arc::new(RwLock::new(HashMap::new())), + Arc::new(crate::NoopStreamInstallProvider), ) } diff --git a/crates/lanspread-peer/src/install/mod.rs b/crates/lanspread-peer/src/install/mod.rs index 4d518d7..e0c404b 100644 --- a/crates/lanspread-peer/src/install/mod.rs +++ b/crates/lanspread-peer/src/install/mod.rs @@ -4,5 +4,13 @@ mod transaction; pub mod unpack; pub use remove::remove_downloaded; -pub use transaction::{install, recover_on_startup, uninstall, update}; +pub(crate) use transaction::root_eti_archives; +pub use transaction::{ + StreamedInstallTransaction, + begin_streamed_install, + install, + recover_on_startup, + uninstall, + update, +}; pub use unpack::{UnpackFuture, Unpacker}; diff --git a/crates/lanspread-peer/src/install/transaction.rs b/crates/lanspread-peer/src/install/transaction.rs index a7ebc85..531cba9 100644 --- a/crates/lanspread-peer/src/install/transaction.rs +++ b/crates/lanspread-peer/src/install/transaction.rs @@ -33,6 +33,144 @@ struct InstallFsState { backup: FsEntryState, } +pub struct StreamedInstallTransaction { + game_root: PathBuf, + state_dir: PathBuf, + id: String, + staging: PathBuf, + eti_version: Option, + created_game_root: bool, +} + +impl StreamedInstallTransaction { + #[must_use] + pub fn staging_dir(&self) -> &Path { + &self.staging + } + + pub async fn commit(self) -> eyre::Result<()> { + let local = local_dir(&self.game_root); + if let Err(err) = tokio::fs::rename(&self.staging, &local) + .await + .wrap_err_with(|| format!("failed to promote streamed install for {}", self.id)) + { + if let Err(cleanup_err) = remove_dir_all_if_exists(&self.staging).await { + log::warn!( + "Failed to clean streamed install staging {}: {cleanup_err}", + self.staging.display() + ); + } + if let Err(cleanup_err) = + remove_created_empty_game_root(&self.game_root, self.created_game_root).await + { + log::warn!( + "Failed to clean streamed install game root {}: {cleanup_err}", + self.game_root.display() + ); + } + let _ = write_intent( + &self.state_dir, + &self.id, + &InstallIntent::none(&self.id, self.eti_version.clone()), + ) + .await; + return Err(err); + } + + if let Err(err) = reset_launch_settings_marker(&self.state_dir, &self.id).await { + log::error!( + "Streamed install for {} was promoted but launch-settings marker reset failed: {err}", + self.id + ); + } + if let Err(err) = write_intent( + &self.state_dir, + &self.id, + &InstallIntent::none(&self.id, self.eti_version.clone()), + ) + .await + { + log::error!( + "Streamed install for {} was promoted but intent cleanup failed: {err}", + self.id + ); + } + + Ok(()) + } + + pub async fn rollback(self) -> eyre::Result<()> { + let cleanup_result = async { + remove_dir_all_if_exists(&self.staging).await?; + remove_created_empty_game_root(&self.game_root, self.created_game_root).await + } + .await; + let intent_result = write_intent( + &self.state_dir, + &self.id, + &InstallIntent::none(&self.id, self.eti_version.clone()), + ) + .await; + + cleanup_result?; + intent_result + } +} + +pub async fn begin_streamed_install( + game_root: &Path, + state_dir: &Path, + id: &str, +) -> eyre::Result { + if path_is_dir(&local_dir(game_root)).await { + eyre::bail!("game {id} is already installed"); + } + + let created_game_root = !path_exists(game_root).await; + tokio::fs::create_dir_all(game_root).await?; + let eti_version = read_downloaded_version(game_root).await; + if let Err(err) = write_intent( + state_dir, + id, + &InstallIntent::new(id, InstallIntentState::Installing, eti_version.clone()), + ) + .await + { + if let Err(cleanup_err) = remove_created_empty_game_root(game_root, created_game_root).await + { + log::warn!( + "Failed to clean streamed install game root {}: {cleanup_err}", + game_root.display() + ); + } + return Err(err); + } + + let staging = installing_dir(game_root); + if let Err(err) = prepare_owned_empty_dir(&staging).await { + let _ = write_intent(state_dir, id, &InstallIntent::none(id, eti_version)).await; + if let Err(cleanup_err) = remove_created_empty_game_root(game_root, created_game_root).await + { + log::warn!( + "Failed to clean streamed install game root {}: {cleanup_err}", + game_root.display() + ); + } + return Err(err); + } + + let staging = tokio::fs::canonicalize(&staging).await.unwrap_or(staging); + + Ok(StreamedInstallTransaction { + game_root: game_root.to_path_buf(), + state_dir: state_dir.to_path_buf(), + id: id.to_string(), + staging, + eti_version, + created_game_root, + }) +} + pub async fn install( game_root: &Path, state_dir: &Path, @@ -258,7 +396,7 @@ async fn unpack_archives( Ok(()) } -async fn root_eti_archives(game_root: &Path) -> eyre::Result> { +pub(crate) async fn root_eti_archives(game_root: &Path) -> eyre::Result> { let mut entries = tokio::fs::read_dir(game_root).await?; let mut archives = Vec::new(); while let Some(entry) = entries.next_entry().await? { @@ -489,6 +627,28 @@ async fn remove_dir_all_if_exists(path: &Path) -> eyre::Result<()> { } } +async fn remove_created_empty_game_root(game_root: &Path, created: bool) -> eyre::Result<()> { + if !created { + return Ok(()); + } + remove_empty_dir_if_exists(game_root).await +} + +async fn remove_empty_dir_if_exists(path: &Path) -> eyre::Result<()> { + match tokio::fs::remove_dir(path).await { + Ok(()) => Ok(()), + Err(err) + if matches!( + err.kind(), + ErrorKind::NotFound | ErrorKind::DirectoryNotEmpty + ) => + { + Ok(()) + } + Err(err) => Err(err.into()), + } +} + async fn path_is_dir(path: &Path) -> bool { tokio::fs::metadata(path) .await @@ -630,6 +790,74 @@ mod tests { assert!(!launch_settings_applied_path(state.path(), "game").exists()); } + #[tokio::test] + async fn streamed_install_rollback_removes_new_empty_game_root() { + let temp = TempDir::new("lanspread-install"); + let state = test_state(); + let root = temp.path().join("streamed-game"); + + let transaction = begin_streamed_install(&root, state.path(), "streamed-game") + .await + .expect("streamed transaction should begin"); + assert!(transaction.staging_dir().is_dir()); + + transaction + .rollback() + .await + .expect("streamed rollback should succeed"); + + assert!(!root.exists()); + let intent = read_intent(state.path(), "streamed-game").await; + assert_eq!(intent.state, InstallIntentState::None); + } + + #[tokio::test] + async fn streamed_install_rollback_keeps_existing_game_root() { + let temp = TempDir::new("lanspread-install"); + let state = test_state(); + let root = temp.game_root(); + write_file(&root.join("version.ini"), b"20250101"); + + let transaction = begin_streamed_install(&root, state.path(), "game") + .await + .expect("streamed transaction should begin"); + + transaction + .rollback() + .await + .expect("streamed rollback should succeed"); + + assert!(root.is_dir()); + assert!(root.join("version.ini").is_file()); + assert!(!root.join(INSTALLING_DIR).exists()); + } + + #[tokio::test] + async fn streamed_install_commit_succeeds_when_post_promote_intent_cleanup_fails() { + let temp = TempDir::new("lanspread-install"); + let state = test_state(); + let root = temp.game_root(); + let transaction = begin_streamed_install(&root, state.path(), "game") + .await + .expect("streamed transaction should begin"); + write_file(&transaction.staging_dir().join("payload.txt"), b"installed"); + + let game_state_dir = crate::state_paths::game_state_dir(state.path(), "game"); + std::fs::remove_dir_all(&game_state_dir).expect("game state dir should be removed"); + write_file(&game_state_dir, b"not a directory"); + + transaction + .commit() + .await + .expect("promoted streamed install should be reported as success"); + + assert_eq!( + std::fs::read(root.join(LOCAL_DIR).join("payload.txt")) + .expect("promoted payload should be present"), + b"installed" + ); + } + #[tokio::test] async fn install_unpacks_multiple_root_eti_archives_in_sorted_order() { let temp = TempDir::new("lanspread-install"); diff --git a/crates/lanspread-peer/src/lib.rs b/crates/lanspread-peer/src/lib.rs index 3d254de..425cca3 100644 --- a/crates/lanspread-peer/src/lib.rs +++ b/crates/lanspread-peer/src/lib.rs @@ -32,6 +32,7 @@ mod remote_peer; mod services; mod startup; mod state_paths; +mod stream_install; #[cfg(test)] mod test_support; @@ -79,9 +80,17 @@ use crate::{ state_paths::resolve_state_dir, }; pub use crate::{ + context::OutboundTransfers, launch_settings::{LaunchSettingsOutcome, apply_launch_settings_once}, startup::PeerRuntimeHandle, state_paths::{launch_settings_applied_path, setup_done_path}, + stream_install::{ + ExternalUnrarStreamProvider, + NoopStreamInstallProvider, + StreamInstallFrameSink, + StreamInstallFuture, + StreamInstallProvider, + }, }; // ============================================================================= @@ -243,6 +252,8 @@ pub enum PeerCommand { file_descriptions: Vec, install_after_download: bool, }, + /// Stream archive-expanded bytes directly into `local/` without keeping root archives. + StreamInstallGame { id: String }, /// Install already-downloaded archives into `local/`. InstallGame { id: String }, /// Remove only the `local/` install for a game. @@ -260,11 +271,29 @@ pub enum PeerCommand { } /// Optional startup settings for non-GUI callers and tests. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Default)] pub struct PeerStartOptions { /// Directory used for peer identity and other state. pub state_dir: Option, pub active_outbound_transfers: Option, + /// Provider used to stream archive entries for low-disk streamed installs. + pub stream_install_provider: Option>, +} + +impl std::fmt::Debug for PeerStartOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PeerStartOptions") + .field("state_dir", &self.state_dir) + .field( + "active_outbound_transfers", + &self.active_outbound_transfers.as_ref().map(|_| "..."), + ) + .field( + "stream_install_provider", + &self.stream_install_provider.as_ref().map(|_| "..."), + ) + .finish() + } } // ============================================================================= @@ -314,11 +343,14 @@ pub fn start_peer_with_options( let PeerStartOptions { state_dir, active_outbound_transfers, + stream_install_provider, } = options; let state_dir = resolve_state_dir(state_dir.as_deref()); let game_dir = game_dir.into(); let active_outbound_transfers = active_outbound_transfers .unwrap_or_else(|| Arc::new(RwLock::new(std::collections::HashMap::new()))); + let stream_install_provider = + stream_install_provider.unwrap_or_else(|| Arc::new(NoopStreamInstallProvider)); log::info!( "Starting peer system with game directory: {}", game_dir.display() @@ -338,6 +370,7 @@ pub fn start_peer_with_options( unpacker, catalog, active_outbound_transfers, + stream_install_provider, )) } @@ -355,6 +388,7 @@ async fn run_peer( task_tracker: TaskTracker, catalog: Arc>, active_outbound_transfers: crate::context::OutboundTransfers, + stream_install_provider: Arc, ) -> eyre::Result<()> { let ctx = Ctx::new( peer_game_db, @@ -366,6 +400,7 @@ async fn run_peer( task_tracker, catalog, active_outbound_transfers, + stream_install_provider, ); if let Err(err) = load_local_library(&ctx, &tx_notify_ui).await { log::error!("Failed to load initial local game database: {err}"); @@ -439,6 +474,9 @@ async fn handle_peer_commands( ) .await; } + PeerCommand::StreamInstallGame { id } => { + handlers::handle_stream_install_game_command(ctx, tx_notify_ui, id).await; + } PeerCommand::InstallGame { id } => { handle_install_game_command(ctx, tx_notify_ui, id).await; } diff --git a/crates/lanspread-peer/src/services/handshake.rs b/crates/lanspread-peer/src/services/handshake.rs index f147582..7ba2aa8 100644 --- a/crates/lanspread-peer/src/services/handshake.rs +++ b/crates/lanspread-peer/src/services/handshake.rs @@ -319,6 +319,7 @@ mod tests { TaskTracker::new(), Arc::new(RwLock::new(catalog)), Arc::new(RwLock::new(HashMap::new())), + Arc::new(crate::NoopStreamInstallProvider), ); *ctx.local_peer_addr.write().await = Some(addr([127, 0, 0, 1], 4000)); diff --git a/crates/lanspread-peer/src/services/local_monitor.rs b/crates/lanspread-peer/src/services/local_monitor.rs index 2885539..f0ce2e9 100644 --- a/crates/lanspread-peer/src/services/local_monitor.rs +++ b/crates/lanspread-peer/src/services/local_monitor.rs @@ -384,6 +384,7 @@ mod tests { TaskTracker::new(), Arc::new(RwLock::new(catalog)), Arc::new(RwLock::new(std::collections::HashMap::new())), + Arc::new(crate::NoopStreamInstallProvider), ) } diff --git a/crates/lanspread-peer/src/services/stream.rs b/crates/lanspread-peer/src/services/stream.rs index 27ae69e..7dee903 100644 --- a/crates/lanspread-peer/src/services/stream.rs +++ b/crates/lanspread-peer/src/services/stream.rs @@ -15,6 +15,7 @@ use crate::{ local_games::{get_game_file_descriptions, is_local_dir_name, local_download_matches_catalog}, peer::{send_game_file_chunk, send_game_file_data}, services::handshake::{HandshakeCtx, accept_inbound_hello, spawn_library_resync}, + stream_install::{send_game_install_stream, send_stream_install_error}, }; type ResponseWriter = FramedWrite; @@ -99,6 +100,9 @@ async fn dispatch_request( } => { handle_file_chunk_request(ctx, game_id, relative_path, offset, length, framed_tx).await } + Request::StreamInstall { game_id } => { + handle_stream_install_request(ctx, game_id, framed_tx).await + } Request::Goodbye { peer_id } => { handle_goodbye(ctx, remote_addr, peer_id).await; framed_tx @@ -386,6 +390,49 @@ async fn handle_file_chunk_request( FramedWrite::new(tx, LengthDelimitedCodec::new()) } +async fn handle_stream_install_request( + ctx: &PeerCtx, + game_id: String, + framed_tx: ResponseWriter, +) -> ResponseWriter { + log::info!("Received StreamInstall request for {game_id} from peer"); + + let (guard, cancel_token) = TransferGuard::new( + game_id.clone(), + ctx.active_outbound_transfers.clone(), + ctx.tx_notify_ui.clone(), + &ctx.shutdown, + ) + .await; + + let mut tx = framed_tx.into_inner(); + let game_dir = ctx.game_dir.read().await.clone(); + if !can_serve_game(ctx, &game_dir, &game_id).await { + log::info!( + "Declining StreamInstall for {game_id} because the game is not currently transferable" + ); + tx = send_stream_install_error(tx, format!("game {game_id} is not transferable")).await; + drop(guard); + return FramedWrite::new(tx, LengthDelimitedCodec::new()); + } + + let game_root = game_dir.join(&game_id); + let (returned_tx, result) = send_game_install_stream( + ctx.stream_install_provider.clone(), + tx, + &game_root, + &game_id, + cancel_token, + ) + .await; + if let Err(err) = result { + log::warn!("StreamInstall for {game_id} ended with error: {err}"); + } + + drop(guard); + FramedWrite::new(returned_tx, LengthDelimitedCodec::new()) +} + async fn handle_goodbye(ctx: &PeerCtx, _remote_addr: Option, peer_id: String) { log::info!("Received Goodbye from peer {peer_id}"); let removed = { ctx.peer_game_db.write().await.remove_peer(&peer_id) }; @@ -442,6 +489,7 @@ mod tests { TaskTracker::new(), Arc::new(RwLock::new(catalog)), Arc::new(RwLock::new(std::collections::HashMap::new())), + Arc::new(crate::NoopStreamInstallProvider), ) .to_peer_ctx(tx_notify_ui) } diff --git a/crates/lanspread-peer/src/startup.rs b/crates/lanspread-peer/src/startup.rs index 2ab4571..96e4479 100644 --- a/crates/lanspread-peer/src/startup.rs +++ b/crates/lanspread-peer/src/startup.rs @@ -23,6 +23,7 @@ use crate::{ PeerCommand, PeerEvent, PeerRuntimeComponent, + StreamInstallProvider, Unpacker, context::Ctx, events, @@ -87,6 +88,7 @@ pub(crate) fn spawn_peer_runtime( unpacker: Arc, catalog: Arc>, active_outbound_transfers: crate::context::OutboundTransfers, + stream_install_provider: Arc, ) -> PeerRuntimeHandle { let shutdown = CancellationToken::new(); let task_tracker = TaskTracker::new(); @@ -107,6 +109,7 @@ pub(crate) fn spawn_peer_runtime( runtime_tracker.clone(), catalog, active_outbound_transfers, + stream_install_provider, ) .await { diff --git a/crates/lanspread-peer/src/stream_install.rs b/crates/lanspread-peer/src/stream_install.rs new file mode 100644 index 0000000..5c93361 --- /dev/null +++ b/crates/lanspread-peer/src/stream_install.rs @@ -0,0 +1,958 @@ +use std::{ + future::Future, + net::SocketAddr, + path::{Path, PathBuf}, + pin::Pin, + process::Stdio, + sync::Arc, + time::{Duration, Instant}, +}; + +use bytes::Bytes; +use crc32fast::Hasher; +use futures::{SinkExt, StreamExt}; +use lanspread_proto::{Message, Request, StreamInstallFrame}; +use s2n_quic::stream::SendStream; +use tokio::{ + fs::File, + io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, + process::Command, + sync::{mpsc, mpsc::UnboundedSender}, + time::{self, MissedTickBehavior}, +}; +use tokio_util::{ + codec::{FramedRead, FramedWrite, LengthDelimitedCodec}, + sync::CancellationToken, +}; + +use crate::{ + DownloadProgress, + PeerEvent, + install::root_eti_archives, + network::connect_to_peer, + path_validation::validate_game_file_path, +}; + +const FRAME_CHANNEL_DEPTH: usize = 16; +const STREAM_INSTALL_PROGRESS_UPDATE_INTERVAL: Duration = Duration::from_millis(500); +const STREAM_CHUNK_SIZE: usize = 256 * 1024; + +/// Integrity metadata advertised by the sender's RAR archive. +/// +/// This catches transport corruption, truncation, and provider bugs. It is not +/// a trusted-content guarantee because a malicious peer controls both the bytes +/// and the archive metadata. Trusted content would need catalog-owned hashes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct SenderArchiveIntegrity { + expected_size: u64, + expected_crc32: u32, +} + +impl SenderArchiveIntegrity { + fn new(expected_size: u64, expected_crc32: u32) -> Self { + Self { + expected_size, + expected_crc32, + } + } + + fn verify(self, relative_path: &str, received: u64, actual_crc32: u32) -> eyre::Result<()> { + if received != self.expected_size { + eyre::bail!( + "streamed file {relative_path} size mismatch: got {received}, expected {}", + self.expected_size + ); + } + + if actual_crc32 != self.expected_crc32 { + eyre::bail!( + "streamed file {relative_path} sender RAR CRC32 mismatch: got {actual_crc32:08X}, expected {:08X}", + self.expected_crc32 + ); + } + + Ok(()) + } +} + +pub type StreamInstallFuture<'a> = Pin> + Send + 'a>>; + +#[derive(Clone)] +pub struct StreamInstallFrameSink { + frames: mpsc::Sender, + cancel_token: CancellationToken, +} + +impl StreamInstallFrameSink { + fn new(frames: mpsc::Sender, cancel_token: CancellationToken) -> Self { + Self { + frames, + cancel_token, + } + } + + pub async fn send(&self, frame: StreamInstallFrame) -> eyre::Result<()> { + tokio::select! { + () = self.cancel_token.cancelled() => { + eyre::bail!("streamed install frame send was cancelled"); + } + result = self.frames.send(frame) => { + result.map_err(|_| eyre::eyre!("streamed install frame receiver closed")) + } + } + } +} + +pub trait StreamInstallProvider: Send + Sync { + fn stream_archive<'a>( + &'a self, + archive: &'a Path, + frames: StreamInstallFrameSink, + cancel_token: CancellationToken, + ) -> StreamInstallFuture<'a>; +} + +#[derive(Debug, Default)] +pub struct NoopStreamInstallProvider; + +impl StreamInstallProvider for NoopStreamInstallProvider { + fn stream_archive<'a>( + &'a self, + archive: &'a Path, + _frames: StreamInstallFrameSink, + _cancel_token: CancellationToken, + ) -> StreamInstallFuture<'a> { + Box::pin(async move { + eyre::bail!( + "streamed install provider is not configured for {}", + archive.display() + ) + }) + } +} + +#[derive(Debug)] +pub struct ExternalUnrarStreamProvider { + program: PathBuf, +} + +impl ExternalUnrarStreamProvider { + #[must_use] + pub fn new(program: PathBuf) -> Self { + Self { program } + } +} + +impl StreamInstallProvider for ExternalUnrarStreamProvider { + fn stream_archive<'a>( + &'a self, + archive: &'a Path, + frames: StreamInstallFrameSink, + cancel_token: CancellationToken, + ) -> StreamInstallFuture<'a> { + Box::pin(async move { + let listing = unrar_listing(&self.program, archive).await?; + let archive_name = archive + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("archive.eti") + .to_string(); + + frames + .send(StreamInstallFrame::ArchiveBegin { + archive_name: archive_name.clone(), + solid: listing.solid, + unpacked_size: listing.unpacked_size(), + }) + .await?; + + stream_unrar_entries( + &self.program, + archive, + &listing.entries, + &frames, + cancel_token.clone(), + ) + .await?; + + frames + .send(StreamInstallFrame::ArchiveEnd { archive_name }) + .await + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct RarListing { + solid: bool, + entries: Vec, +} + +impl RarListing { + fn unpacked_size(&self) -> u64 { + self.entries + .iter() + .filter(|entry| entry.kind == RarEntryKind::File) + .map(|entry| entry.size) + .sum() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct RarEntry { + relative_path: String, + kind: RarEntryKind, + size: u64, + crc32: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RarEntryKind { + File, + Directory, +} + +#[derive(Default)] +struct RarEntryDraft { + relative_path: Option, + kind: Option, + size: Option, + crc32: Option, +} + +async fn unrar_listing(program: &Path, archive: &Path) -> eyre::Result { + let output = Command::new(program) + .arg("lt") + .arg("-cfg-") + .arg(archive) + .output() + .await?; + if !output.status.success() { + eyre::bail!( + "unrar lt failed for {} with status {}: {}", + archive.display(), + output.status, + String::from_utf8_lossy(&output.stderr) + ); + } + + parse_unrar_listing(&String::from_utf8_lossy(&output.stdout)) +} + +fn parse_unrar_listing(output: &str) -> eyre::Result { + let mut solid = false; + let mut entries = Vec::new(); + let mut current = RarEntryDraft::default(); + + for line in output.lines() { + let trimmed = line.trim(); + if let Some(details) = trimmed.strip_prefix("Details:") { + solid = details.to_ascii_lowercase().contains("solid"); + continue; + } + + if let Some(name) = trimmed.strip_prefix("Name:") { + push_rar_entry(&mut entries, std::mem::take(&mut current))?; + current.relative_path = Some(name.trim().to_string()); + continue; + } + + if let Some(kind) = trimmed.strip_prefix("Type:") { + current.kind = match kind.trim() { + "File" => Some(RarEntryKind::File), + "Directory" => Some(RarEntryKind::Directory), + _ => None, + }; + continue; + } + + if let Some(size) = trimmed.strip_prefix("Size:") { + current.size = Some(size.trim().parse()?); + continue; + } + + if let Some(crc) = trimmed.strip_prefix("CRC32:") { + current.crc32 = Some(u32::from_str_radix(crc.trim(), 16)?); + } + } + + push_rar_entry(&mut entries, current)?; + Ok(RarListing { solid, entries }) +} + +fn push_rar_entry(entries: &mut Vec, draft: RarEntryDraft) -> eyre::Result<()> { + let Some(relative_path) = draft.relative_path else { + return Ok(()); + }; + + let Some(kind) = draft.kind else { + return Ok(()); + }; + + let (size, crc32) = match kind { + RarEntryKind::File => { + let size = draft + .size + .ok_or_else(|| eyre::eyre!("RAR file entry {relative_path} has no Size"))?; + let crc32 = match (size, draft.crc32) { + (_, Some(crc32)) => crc32, + (0, None) => 0, + (_, None) => { + eyre::bail!("RAR file entry {relative_path} has no CRC32"); + } + }; + (size, Some(crc32)) + } + RarEntryKind::Directory => (0, None), + }; + + entries.push(RarEntry { + relative_path, + kind, + size, + crc32, + }); + Ok(()) +} + +async fn stream_unrar_entries( + program: &Path, + archive: &Path, + entries: &[RarEntry], + frames: &StreamInstallFrameSink, + cancel_token: CancellationToken, +) -> eyre::Result<()> { + let mut child = Command::new(program) + .arg("p") + .arg("-inul") + .arg("-cfg-") + .arg(archive) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + + let result = async { + let mut stdout = child + .stdout + .take() + .ok_or_else(|| eyre::eyre!("unrar stdout was not captured"))?; + let mut buffer = vec![0_u8; STREAM_CHUNK_SIZE]; + + for entry in entries { + if cancel_token.is_cancelled() { + eyre::bail!("streamed archive {} was cancelled", archive.display()); + } + + match entry.kind { + RarEntryKind::Directory => { + frames + .send(StreamInstallFrame::Directory { + relative_path: entry.relative_path.clone(), + }) + .await?; + } + RarEntryKind::File => { + let Some(crc32) = entry.crc32 else { + eyre::bail!("RAR file entry {} has no CRC32", entry.relative_path); + }; + frames + .send(StreamInstallFrame::FileBegin { + relative_path: entry.relative_path.clone(), + size: entry.size, + crc32, + }) + .await?; + stream_unrar_file_from_stdout( + &mut stdout, + archive, + entry, + frames, + &mut buffer, + &cancel_token, + ) + .await?; + frames + .send(StreamInstallFrame::FileEnd { + relative_path: entry.relative_path.clone(), + }) + .await?; + } + } + } + + let extra = + read_unrar_stdout(&mut stdout, &mut buffer[..1], &cancel_token, archive).await?; + if extra != 0 { + eyre::bail!( + "unrar produced bytes after listed entries for {}", + archive.display() + ); + } + + let status = wait_unrar_child(&mut child, &cancel_token, archive).await?; + if !status.success() { + eyre::bail!( + "unrar p failed for {} with status {status}", + archive.display() + ); + } + + Ok(()) + } + .await; + + if result.is_err() { + let _ = child.kill().await; + } + + result +} + +async fn stream_unrar_file_from_stdout( + stdout: &mut (impl AsyncRead + Unpin), + archive: &Path, + entry: &RarEntry, + frames: &StreamInstallFrameSink, + buffer: &mut [u8], + cancel_token: &CancellationToken, +) -> eyre::Result<()> { + let mut remaining = entry.size; + while remaining > 0 { + let read_len = usize::try_from(remaining.min(u64::try_from(buffer.len())?))?; + let read = + read_unrar_stdout(stdout, &mut buffer[..read_len], cancel_token, archive).await?; + if read == 0 { + eyre::bail!( + "unrar ended while streaming {} from {}; {remaining} bytes missing", + entry.relative_path, + archive.display() + ); + } + + frames + .send(StreamInstallFrame::FileChunk { + bytes: Bytes::copy_from_slice(&buffer[..read]), + }) + .await?; + remaining = remaining.saturating_sub(u64::try_from(read)?); + } + + Ok(()) +} + +async fn read_unrar_stdout( + stdout: &mut (impl AsyncRead + Unpin), + buffer: &mut [u8], + cancel_token: &CancellationToken, + archive: &Path, +) -> eyre::Result { + tokio::select! { + () = cancel_token.cancelled() => { + eyre::bail!("streamed archive {} was cancelled", archive.display()); + } + read = stdout.read(buffer) => Ok(read?), + } +} + +async fn wait_unrar_child( + child: &mut tokio::process::Child, + cancel_token: &CancellationToken, + archive: &Path, +) -> eyre::Result { + tokio::select! { + () = cancel_token.cancelled() => { + let _ = child.kill().await; + eyre::bail!("streamed archive {} was cancelled", archive.display()); + } + status = child.wait() => Ok(status?), + } +} + +pub(crate) async fn send_stream_install_error( + tx: SendStream, + message: impl Into, +) -> SendStream { + let mut framed_tx = FramedWrite::new(tx, LengthDelimitedCodec::new()); + if let Err(err) = framed_tx + .send( + StreamInstallFrame::Error { + message: message.into(), + } + .encode(), + ) + .await + { + log::warn!("Failed to send streamed install error frame: {err}"); + } + if let Err(err) = framed_tx.close().await { + log::debug!("Failed to close streamed install error response: {err}"); + } + framed_tx.into_inner() +} + +pub(crate) async fn send_game_install_stream( + provider: Arc, + tx: SendStream, + game_root: &Path, + game_id: &str, + cancel_token: CancellationToken, +) -> (SendStream, eyre::Result<()>) { + let archives = match root_eti_archives(game_root).await { + Ok(archives) => archives, + Err(err) => { + let message = err.to_string(); + let tx = send_stream_install_error(tx, message.clone()).await; + return (tx, Err(eyre::eyre!(message))); + } + }; + if archives.is_empty() { + let message = format!("no .eti archives found for {game_id}"); + let tx = send_stream_install_error(tx, message.clone()).await; + return (tx, Err(eyre::eyre!(message))); + } + + let (frame_tx, mut frame_rx) = mpsc::channel(FRAME_CHANNEL_DEPTH); + let producer_cancel = cancel_token.child_token(); + let frame_sink = StreamInstallFrameSink::new(frame_tx, producer_cancel.clone()); + let game_id_for_producer = game_id.to_string(); + let producer = tokio::spawn({ + let provider = provider.clone(); + let producer_cancel = producer_cancel.clone(); + async move { + for archive in archives { + if producer_cancel.is_cancelled() { + eyre::bail!("streamed install for {game_id_for_producer} was cancelled"); + } + + if let Err(err) = provider + .stream_archive(&archive, frame_sink.clone(), producer_cancel.clone()) + .await + { + let message = err.to_string(); + let _ = frame_sink.send(StreamInstallFrame::Error { message }).await; + return Err(err); + } + } + + let _ = frame_sink.send(StreamInstallFrame::Complete).await; + Ok(()) + } + }); + + let mut framed_tx = FramedWrite::new(tx, LengthDelimitedCodec::new()); + let mut send_result = Ok(()); + + while let Some(frame) = frame_rx.recv().await { + if let Err(err) = framed_tx.send(frame.encode()).await { + producer_cancel.cancel(); + send_result = Err(eyre::eyre!("failed to send streamed install frame: {err}")); + break; + } + } + drop(frame_rx); + + let close_result = framed_tx + .close() + .await + .map_err(|err| eyre::eyre!("failed to close streamed install stream: {err}")); + let tx = framed_tx.into_inner(); + let producer_result = match producer.await { + Ok(result) => result, + Err(err) => Err(eyre::eyre!("streamed install producer task failed: {err}")), + }; + let result = send_result.and(producer_result).and(close_result); + + (tx, result) +} + +pub(crate) async fn receive_streamed_install( + peer_addr: SocketAddr, + game_id: &str, + staging_dir: &Path, + tx_notify_ui: UnboundedSender, + cancel_token: CancellationToken, +) -> eyre::Result<()> { + let staging_dir = tokio::fs::canonicalize(staging_dir) + .await + .unwrap_or_else(|_| staging_dir.to_path_buf()); + let mut conn = connect_to_peer(peer_addr).await?; + let stream = conn.open_bidirectional_stream().await?; + let (rx, tx) = stream.split(); + let mut framed_tx = FramedWrite::new(tx, LengthDelimitedCodec::new()); + + framed_tx + .send( + Request::StreamInstall { + game_id: game_id.to_string(), + } + .encode(), + ) + .await?; + framed_tx.close().await?; + + let mut framed_rx = FramedRead::new(rx, LengthDelimitedCodec::new()); + let mut current_file: Option = None; + let mut progress = StreamInstallProgress::new(game_id.to_string()); + let mut progress_interval = time::interval(STREAM_INSTALL_PROGRESS_UPDATE_INTERVAL); + progress_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + progress_interval.tick().await; + + loop { + let next = tokio::select! { + () = cancel_token.cancelled() => eyre::bail!("streamed install for {game_id} was cancelled"), + _ = progress_interval.tick() => { + progress.emit_current(&tx_notify_ui); + continue; + } + next = framed_rx.next() => next, + }; + + let Some(frame) = next else { + eyre::bail!("streamed install ended before Complete"); + }; + let frame = frame?.freeze(); + let frame = StreamInstallFrame::decode(frame); + + match frame { + StreamInstallFrame::ArchiveBegin { + archive_name, + solid, + unpacked_size, + } => { + progress.add_total(unpacked_size); + progress.emit_snapshot(&tx_notify_ui, 0); + log::info!( + "Receiving streamed install archive {archive_name} for {game_id} \ + (solid={solid}, unpacked_size={unpacked_size})" + ); + } + StreamInstallFrame::Directory { relative_path } => { + let path = resolve_stream_path(&staging_dir, &relative_path)?; + tokio::fs::create_dir_all(path).await?; + } + StreamInstallFrame::FileBegin { + relative_path, + size, + crc32, + } => { + if current_file.is_some() { + eyre::bail!("received FileBegin for {relative_path} before previous FileEnd"); + } + let path = resolve_stream_path(&staging_dir, &relative_path)?; + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let file = File::create(&path).await?; + current_file = Some(IncomingFile::new(relative_path, path, size, crc32, file)); + } + StreamInstallFrame::FileChunk { bytes } => { + let Some(file) = current_file.as_mut() else { + eyre::bail!("received FileChunk without FileBegin"); + }; + let length = file + .write_chunk(game_id, peer_addr, &tx_notify_ui, bytes) + .await?; + progress.record_bytes(length); + } + StreamInstallFrame::FileEnd { relative_path } => { + let Some(file) = current_file.take() else { + eyre::bail!("received FileEnd for {relative_path} without FileBegin"); + }; + file.finish(&relative_path).await?; + } + StreamInstallFrame::ArchiveEnd { archive_name } => { + log::info!("Finished streamed install archive {archive_name} for {game_id}"); + } + StreamInstallFrame::Complete => { + if current_file.is_some() { + eyre::bail!("streamed install completed with an open file"); + } + progress.emit_snapshot(&tx_notify_ui, 0); + return Ok(()); + } + StreamInstallFrame::Error { message } => { + eyre::bail!("streamed install sender failed: {message}"); + } + } + } +} + +struct StreamInstallProgress { + id: String, + total_bytes: u64, + downloaded_bytes: u64, + last_downloaded_bytes: u64, + last_at: Instant, +} + +impl StreamInstallProgress { + fn new(id: String) -> Self { + Self { + id, + total_bytes: 0, + downloaded_bytes: 0, + last_downloaded_bytes: 0, + last_at: Instant::now(), + } + } + + fn add_total(&mut self, bytes: u64) { + self.total_bytes = self.total_bytes.saturating_add(bytes); + } + + fn record_bytes(&mut self, bytes: u64) { + self.downloaded_bytes = self.downloaded_bytes.saturating_add(bytes); + } + + fn emit_current(&mut self, tx_notify_ui: &UnboundedSender) { + let now = Instant::now(); + let speed = bytes_per_second( + self.downloaded_bytes + .saturating_sub(self.last_downloaded_bytes), + now.duration_since(self.last_at), + ); + + self.last_downloaded_bytes = self.downloaded_bytes; + self.last_at = now; + self.emit_snapshot(tx_notify_ui, speed); + } + + fn emit_snapshot(&self, tx_notify_ui: &UnboundedSender, bytes_per_second: u64) { + let _ = tx_notify_ui.send(PeerEvent::DownloadGameFilesProgress(DownloadProgress { + id: self.id.clone(), + downloaded_bytes: self.downloaded_bytes, + total_bytes: self.total_bytes, + bytes_per_second, + active_peer_count: 1, + })); + } +} + +fn bytes_per_second(bytes: u64, elapsed: Duration) -> u64 { + let millis = elapsed.as_millis().max(1); + let rate = u128::from(bytes).saturating_mul(1_000) / millis; + u64::try_from(rate).unwrap_or(u64::MAX) +} + +struct IncomingFile { + relative_path: String, + path: PathBuf, + integrity: SenderArchiveIntegrity, + received: u64, + crc32: Hasher, + file: File, +} + +impl IncomingFile { + fn new( + relative_path: String, + path: PathBuf, + expected_size: u64, + expected_crc32: u32, + file: File, + ) -> Self { + Self { + relative_path, + path, + integrity: SenderArchiveIntegrity::new(expected_size, expected_crc32), + received: 0, + crc32: Hasher::new(), + file, + } + } + + async fn write_chunk( + &mut self, + game_id: &str, + peer_addr: SocketAddr, + tx_notify_ui: &UnboundedSender, + bytes: Bytes, + ) -> eyre::Result { + let offset = self.received; + let length = u64::try_from(bytes.len())?; + if offset.saturating_add(length) > self.integrity.expected_size { + eyre::bail!( + "streamed file {} exceeded expected size {}", + self.relative_path, + self.integrity.expected_size + ); + } + self.file.write_all(&bytes).await?; + self.crc32.update(&bytes); + self.received = self.received.saturating_add(length); + + let _ = tx_notify_ui.send(PeerEvent::DownloadGameFileChunkFinished { + id: game_id.to_string(), + peer_addr, + relative_path: format!("{game_id}/.local.installing/{}", self.relative_path), + offset, + length, + }); + Ok(length) + } + + async fn finish(mut self, relative_path: &str) -> eyre::Result<()> { + if self.relative_path != relative_path { + eyre::bail!( + "streamed file end mismatch: began {}, ended {relative_path}", + self.relative_path + ); + } + self.file.flush().await?; + + let actual_crc32 = self.crc32.finalize(); + self.integrity + .verify(&self.relative_path, self.received, actual_crc32)?; + + log::debug!( + "Received streamed file {} -> {}", + self.relative_path, + self.path.display() + ); + Ok(()) + } +} + +fn resolve_stream_path(staging_dir: &Path, relative_path: &str) -> eyre::Result { + validate_game_file_path(staging_dir, relative_path) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::TempDir; + + #[test] + fn stream_paths_stay_inside_staging_dir() { + let temp = TempDir::new("lanspread-stream-install-path"); + let staging = temp.path().join("staging"); + std::fs::create_dir_all(&staging).expect("staging should be created"); + let staging = std::fs::canonicalize(staging).expect("staging should canonicalize"); + + assert!(resolve_stream_path(&staging, "bin/game.exe").is_ok()); + assert!(resolve_stream_path(&staging, "../outside").is_err()); + assert!(resolve_stream_path(&staging, "/absolute").is_err()); + assert!(resolve_stream_path(&staging, "C:/windows").is_err()); + } + + #[test] + fn parses_unrar_technical_listing() { + let listing = parse_unrar_listing( + r#" +Archive: game.eti +Details: RAR 5, solid + + Name: bin/payload.bin + Type: File + Size: 123 + CRC32: 38B488A7 + + Name: bin + Type: Directory +"#, + ) + .expect("listing should parse"); + + assert!(listing.solid); + assert_eq!( + listing.entries, + vec![ + RarEntry { + relative_path: "bin/payload.bin".to_string(), + kind: RarEntryKind::File, + size: 123, + crc32: Some(0x38B4_88A7), + }, + RarEntry { + relative_path: "bin".to_string(), + kind: RarEntryKind::Directory, + size: 0, + crc32: None, + }, + ] + ); + } + + #[test] + fn rejects_unrar_file_entries_without_crc32() { + let err = parse_unrar_listing( + r#" +Archive: game.eti +Details: RAR 5 + + Name: bin/payload.bin + Type: File + Size: 123 +"#, + ) + .expect_err("file entries without CRC32 should be rejected"); + + assert!(err.to_string().contains("has no CRC32")); + } + + #[test] + fn accepts_zero_size_unrar_file_entries_without_crc32() { + let listing = parse_unrar_listing( + r#" +Archive: game.eti +Details: RAR 5 + + Name: bin/empty.cfg + Type: File + Size: 0 +"#, + ) + .expect("empty file without CRC32 should parse as CRC32 zero"); + + assert_eq!( + listing.entries, + vec![RarEntry { + relative_path: "bin/empty.cfg".to_string(), + kind: RarEntryKind::File, + size: 0, + crc32: Some(0), + }] + ); + } + + #[test] + fn sender_archive_integrity_accepts_matching_size_and_crc32() { + let bytes = b"payload"; + let integrity = + SenderArchiveIntegrity::new(u64::try_from(bytes.len()).unwrap(), crc32_of(bytes)); + + integrity + .verify( + "bin/payload.bin", + u64::try_from(bytes.len()).unwrap(), + crc32_of(bytes), + ) + .expect("matching sender archive metadata should verify"); + } + + #[test] + fn sender_archive_integrity_rejects_size_mismatch() { + let integrity = SenderArchiveIntegrity::new(7, crc32_of(b"payload")); + let err = integrity + .verify("bin/payload.bin", 6, crc32_of(b"payload")) + .expect_err("truncated file should fail sender archive integrity"); + + assert!(err.to_string().contains("size mismatch")); + } + + #[test] + fn sender_archive_integrity_rejects_crc32_mismatch() { + let integrity = SenderArchiveIntegrity::new(7, crc32_of(b"payload")); + let err = integrity + .verify("bin/payload.bin", 7, crc32_of(b"paylord")) + .expect_err("mutated file should fail sender archive integrity"); + + assert!(err.to_string().contains("sender RAR CRC32 mismatch")); + } + + fn crc32_of(bytes: &[u8]) -> u32 { + let mut hasher = Hasher::new(); + hasher.update(bytes); + hasher.finalize() + } +} diff --git a/crates/lanspread-proto/src/lib.rs b/crates/lanspread-proto/src/lib.rs index ea8f2f7..3aad2dc 100644 --- a/crates/lanspread-proto/src/lib.rs +++ b/crates/lanspread-proto/src/lib.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use lanspread_db::db::{Game, GameFileDescription}; use serde::{Deserialize, Serialize}; -pub const PROTOCOL_VERSION: u32 = 4; +pub const PROTOCOL_VERSION: u32 = 5; pub use lanspread_db::db::Availability; @@ -67,6 +67,9 @@ pub enum Request { offset: u64, length: u64, }, + StreamInstall { + game_id: String, + }, Hello(Hello), LibraryDelta { peer_id: String, @@ -94,6 +97,41 @@ pub enum Response { InternalPeerError(String), } +const STREAM_INSTALL_CONTROL_FRAME_TAG: u8 = 0; +const STREAM_INSTALL_FILE_CHUNK_FRAME_TAG: u8 = 1; +const STREAM_INSTALL_ENCODE_ERROR_FRAME: &[u8] = + b"\0{\"Error\":{\"message\":\"stream install frame encoding error\"}}"; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum StreamInstallFrame { + ArchiveBegin { + archive_name: String, + solid: bool, + unpacked_size: u64, + }, + Directory { + relative_path: String, + }, + FileBegin { + relative_path: String, + size: u64, + crc32: u32, + }, + FileChunk { + bytes: Bytes, + }, + FileEnd { + relative_path: String, + }, + ArchiveEnd { + archive_name: String, + }, + Complete, + Error { + message: String, + }, +} + // Add Message trait pub trait Message { fn decode(bytes: Bytes) -> Self; @@ -145,3 +183,62 @@ impl Message for Response { } } } + +impl Message for StreamInstallFrame { + fn decode(bytes: Bytes) -> Self { + if bytes.is_empty() { + return stream_install_decode_error("stream install frame is empty"); + } + + let tag = bytes[0]; + let payload = bytes.slice(1..); + match tag { + STREAM_INSTALL_CONTROL_FRAME_TAG => decode_stream_install_control_frame(&payload), + STREAM_INSTALL_FILE_CHUNK_FRAME_TAG => StreamInstallFrame::FileChunk { bytes: payload }, + _ => stream_install_decode_error(format!("unknown stream install frame tag {tag}")), + } + } + + fn encode(&self) -> Bytes { + match self { + StreamInstallFrame::FileChunk { bytes } => { + tagged_stream_install_frame(STREAM_INSTALL_FILE_CHUNK_FRAME_TAG, bytes) + } + _ => match serde_json::to_vec(self) { + Ok(payload) => { + tagged_stream_install_frame(STREAM_INSTALL_CONTROL_FRAME_TAG, &payload) + } + Err(e) => { + tracing::error!(?e, "StreamInstallFrame encoding error"); + Bytes::from_static(STREAM_INSTALL_ENCODE_ERROR_FRAME) + } + }, + } + } +} + +fn decode_stream_install_control_frame(payload: &[u8]) -> StreamInstallFrame { + match serde_json::from_slice(payload) { + Ok(StreamInstallFrame::FileChunk { .. }) => { + stream_install_decode_error("stream install control frame cannot contain file bytes") + } + Ok(frame) => frame, + Err(e) => { + tracing::error!(?e, "StreamInstallFrame decoding error"); + stream_install_decode_error(format!("stream install frame decoding error: {e}")) + } + } +} + +fn tagged_stream_install_frame(tag: u8, payload: &[u8]) -> Bytes { + let mut frame = Vec::with_capacity(1 + payload.len()); + frame.push(tag); + frame.extend_from_slice(payload); + Bytes::from(frame) +} + +fn stream_install_decode_error(message: impl Into) -> StreamInstallFrame { + StreamInstallFrame::Error { + message: message.into(), + } +} diff --git a/crates/lanspread-proto/tests/stream_install_frame.rs b/crates/lanspread-proto/tests/stream_install_frame.rs new file mode 100644 index 0000000..ed44eb7 --- /dev/null +++ b/crates/lanspread-proto/tests/stream_install_frame.rs @@ -0,0 +1,42 @@ +use bytes::Bytes; +use lanspread_proto::{Message, StreamInstallFrame}; + +#[test] +fn file_chunks_encode_raw_bytes() { + let bytes = Bytes::from_static(&[0, 1, 2, 255]); + let encoded = StreamInstallFrame::FileChunk { + bytes: bytes.clone(), + } + .encode(); + + assert_eq!(&encoded[..], &[1, 0, 1, 2, 255]); + assert_eq!( + StreamInstallFrame::decode(encoded), + StreamInstallFrame::FileChunk { bytes } + ); +} + +#[test] +fn control_frames_are_tagged_json() { + let frame = StreamInstallFrame::FileBegin { + relative_path: "bin/game.exe".to_string(), + size: 42, + crc32: 0x38B4_88A7, + }; + let encoded = frame.encode(); + + assert_eq!(encoded[0], 0); + assert_eq!(StreamInstallFrame::decode(encoded), frame); +} + +#[test] +fn empty_frames_decode_as_errors() { + match StreamInstallFrame::decode(Bytes::new()) { + StreamInstallFrame::Error { message } => { + assert!(message.contains("empty")); + } + other => { + panic!("expected error frame, got {other:?}"); + } + } +} diff --git a/crates/lanspread-tauri-deno-ts/src-tauri/src/lib.rs b/crates/lanspread-tauri-deno-ts/src-tauri/src/lib.rs index 2a9fa11..b6fb5f2 100644 --- a/crates/lanspread-tauri-deno-ts/src-tauri/src/lib.rs +++ b/crates/lanspread-tauri-deno-ts/src-tauri/src/lib.rs @@ -14,11 +14,14 @@ use lanspread_db::db::{Availability, Game, GameCatalog, GameDB, GameFileDescript use lanspread_peer::{ ActiveOperation, ActiveOperationKind, + ExternalUnrarStreamProvider, + NoopStreamInstallProvider, PeerCommand, PeerEvent, PeerGameDB, PeerRuntimeHandle, PeerStartOptions, + StreamInstallProvider, UnpackFuture, Unpacker, migrate_legacy_state, @@ -294,6 +297,56 @@ async fn install_game( Ok(handled) } +#[tauri::command] +async fn stream_install_game( + id: String, + state: tauri::State<'_, LanSpreadState>, +) -> tauri::Result { + if state + .inner() + .active_operations + .read() + .await + .contains_key(&id) + { + log::warn!("Game already has an active operation: {id}"); + return Ok(false); + } + + let Some((downloaded, installed, peer_count)) = state + .inner() + .games + .read() + .await + .get_game_by_id(&id) + .map(|game| (game.downloaded, game.installed, game.peer_count)) + else { + log::warn!("Ignoring streamed install request for unknown game: {id}"); + return Ok(false); + }; + if downloaded || installed || peer_count == 0 { + log::warn!( + "Ignoring streamed install request for {id}: downloaded={downloaded}, \ + installed={installed}, peer_count={peer_count}" + ); + return Ok(false); + } + + let peer_ctrl_arc = state.inner().peer_ctrl.clone(); + let peer_ctrl = peer_ctrl_arc.read().await.clone(); + let Some(peer_ctrl) = peer_ctrl else { + log::warn!("Peer system not initialized yet"); + return Ok(false); + }; + + if let Err(e) = peer_ctrl.send(PeerCommand::StreamInstallGame { id }) { + log::error!("Failed to send PeerCommand::StreamInstallGame: {e:?}"); + return Ok(false); + } + + Ok(true) +} + #[tauri::command] async fn update_game( id: String, @@ -1867,6 +1920,7 @@ async fn ensure_peer_started(app_handle: &AppHandle, games_folder: &Path) { let unpacker = Arc::new(SidecarUnpacker { app_handle: app_handle.clone(), }); + let stream_install_provider = stream_install_provider_for_app(app_handle); match start_peer_with_options( games_folder.to_path_buf(), tx_peer_event, @@ -1876,6 +1930,7 @@ async fn ensure_peer_started(app_handle: &AppHandle, games_folder: &Path) { PeerStartOptions { state_dir: Some(state_dir), active_outbound_transfers: Some(state.active_outbound_transfers.clone()), + stream_install_provider: Some(stream_install_provider), }, ) { Ok(handle) => { @@ -1893,6 +1948,22 @@ async fn ensure_peer_started(app_handle: &AppHandle, games_folder: &Path) { } } +fn stream_install_provider_for_app(app_handle: &AppHandle) -> Arc { + match resolve_unrar_sidecar_program(app_handle) { + Ok(program) => Arc::new(ExternalUnrarStreamProvider::new(program)), + Err(err) => { + log::error!("Failed to resolve streamed-install unrar sidecar: {err}"); + Arc::new(NoopStreamInstallProvider) + } + } +} + +fn resolve_unrar_sidecar_program(app_handle: &AppHandle) -> eyre::Result { + let sidecar = app_handle.shell().sidecar("unrar")?; + let command: std::process::Command = sidecar.into(); + Ok(PathBuf::from(command.get_program())) +} + fn emit_game_id_event(app_handle: &AppHandle, event: &str, id: &str, label: &str) { if let Err(e) = app_handle.emit(event, Some(id.to_owned())) { log::error!("{label}: Failed to emit {event} event: {e}"); @@ -2181,7 +2252,7 @@ async fn handle_got_game_files( file_descriptions, }) { - log::error!("Failed to send PeerCommand::DownloadGameFiles: {e}"); + log::error!("Failed to continue queued game transfer: {e}"); } } @@ -2678,6 +2749,7 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ request_games, install_game, + stream_install_game, run_game, start_server, game_directory_exists, diff --git a/crates/lanspread-tauri-deno-ts/src/components/StateChip.tsx b/crates/lanspread-tauri-deno-ts/src/components/StateChip.tsx index 9c49213..214b563 100644 --- a/crates/lanspread-tauri-deno-ts/src/components/StateChip.tsx +++ b/crates/lanspread-tauri-deno-ts/src/components/StateChip.tsx @@ -1,13 +1,5 @@ import { Game } from '../lib/types'; -import { deriveState } from '../lib/gameState'; - -const LABELS: Record = { - installed: 'Installed', - local: 'Local', - downloading: 'Downloading', - busy: 'Working', - none: '', -}; +import { deriveState, stateChipLabel } from '../lib/gameState'; interface Props { game: Game; @@ -17,7 +9,7 @@ interface Props { export const StateChip = ({ game, showNone = false }: Props) => { const state = deriveState(game); - const label = LABELS[state] ?? ''; + const label = stateChipLabel(game); if (!label && !showNone) return null; return (
diff --git a/crates/lanspread-tauri-deno-ts/src/components/modals/GameDetailModal.tsx b/crates/lanspread-tauri-deno-ts/src/components/modals/GameDetailModal.tsx index 96e5e87..117ba36 100644 --- a/crates/lanspread-tauri-deno-ts/src/components/modals/GameDetailModal.tsx +++ b/crates/lanspread-tauri-deno-ts/src/components/modals/GameDetailModal.tsx @@ -5,7 +5,7 @@ import { StateChip } from '../StateChip'; import { ActionButton } from '../ActionButton'; import { Game, InstallStatus } from '../../lib/types'; -import { deriveState, hasNewerLocalVersion, isInProgress } from '../../lib/gameState'; +import { canStreamInstall, gameStatusLabel, hasNewerLocalVersion, isInProgress } from '../../lib/gameState'; import { formatBytes, formatEtiVersion, formatPlayers } from '../../lib/format'; interface Props { @@ -13,6 +13,7 @@ interface Props { thumbnailUrl: string | null; onClose: () => void; onPrimary: (game: Game) => void; + onStreamInstall: (game: Game) => void; onUninstall: (game: Game) => void; onRemoveDownload: (game: Game) => void; onCancelDownload: (game: Game) => void; @@ -28,21 +29,12 @@ const tagsFromGame = (game: Game): string[] => { return tags; }; -const statusLabelFor = (game: Game): string => { - switch (deriveState(game)) { - case 'installed': return 'Installed'; - case 'local': return 'Downloaded'; - case 'downloading': return 'Downloading'; - case 'busy': return 'Working…'; - case 'none': return 'Not downloaded'; - } -}; - export const GameDetailModal = ({ game, thumbnailUrl, onClose, onPrimary, + onStreamInstall, onUninstall, onRemoveDownload, onCancelDownload, @@ -55,6 +47,7 @@ export const GameDetailModal = ({ const canRemoveDownload = game.downloaded && !game.installed && !isInProgress(game.install_status); + const showStreamInstall = canStreamInstall(game); const canViewFiles = game.downloaded || game.installed || game.install_status === InstallStatus.Downloading @@ -112,7 +105,7 @@ export const GameDetailModal = ({
Status
-
{statusLabelFor(game)}
+
{gameStatusLabel(game)}
@@ -133,6 +126,17 @@ export const GameDetailModal = ({ onClick={() => onPrimary(game)} onCancelDownload={onCancelDownload} /> + {showStreamInstall && ( + + )} {game.installed && game.can_host_server === true && (