From d7f7dc737e06bed7208a5586a3b8b71ac0240d40 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Wed, 20 May 2026 08:38:29 +0200 Subject: [PATCH] perf(peer): request larger QUIC UDP socket buffers Configure the s2n-quic Tokio IO provider on both client and server instead of using the address-only default provider. The configured provider asks the OS for 4 MiB send and receive buffers on each QUIC UDP socket, which avoids starting bulk LAN transfers on the tiny default UDP buffer sizes. I tested a wider version that also raised s2n-quic internal IO queues to 8 MiB, but that regressed S37 to 710.19 and 736.20 MiB/s in repeat runs. This commit keeps the narrower socket-buffer request, which measured faster than the prior flow-control-only tuning while leaving the internal queue defaults intact. The host used for measurement reports: - net.core.rmem_max = 16777216 - net.core.wmem_max = 16777216 - net.core.rmem_default = 212992 - net.core.wmem_default = 212992 S37 single-source throughput: - Step 1: 824.94 MiB/s, 6920.09 Mbit/s, 2.483s - Step 2 sample A: 848.15 MiB/s, 7114.81 Mbit/s, 2.415s - Step 2 sample B: 874.06 MiB/s, 7332.12 Mbit/s, 2.343s Test Plan: - just fmt - sysctl net.core.rmem_max net.core.wmem_max net.core.rmem_default \ net.core.wmem_default - python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py S37 --build-image - python3 crates/lanspread-peer-cli/scripts/run_extended_scenarios.py S37 Refs: local LAN download performance investigation on 2026-05-20. Depends-on: cd8bcbfeedfa (QUIC flow-control and BBR tuning). --- crates/lanspread-peer/src/config.rs | 3 +++ crates/lanspread-peer/src/network.rs | 17 +++++++++++++++-- crates/lanspread-peer/src/services/server.rs | 4 ++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/lanspread-peer/src/config.rs b/crates/lanspread-peer/src/config.rs index e20ff5c..b8473f1 100644 --- a/crates/lanspread-peer/src/config.rs +++ b/crates/lanspread-peer/src/config.rs @@ -35,6 +35,9 @@ pub const QUIC_MAX_SEND_BUFFER_SIZE: u32 = 32 * 1024 * 1024; /// Initial congestion window for LAN-oriented BBR transfers (1 MiB). pub const QUIC_INITIAL_CONGESTION_WINDOW: u32 = 1024 * 1024; +/// Requested OS UDP send and receive buffer size for QUIC sockets (4 MiB). +pub const QUIC_SOCKET_BUFFER_SIZE: usize = 4 * 1024 * 1024; + /// Fallback interval for reconciling missed filesystem watcher events (seconds). pub const LOCAL_GAME_FALLBACK_SCAN_SECS: u64 = 300; diff --git a/crates/lanspread-peer/src/network.rs b/crates/lanspread-peer/src/network.rs index c96b066..7609902 100644 --- a/crates/lanspread-peer/src/network.rs +++ b/crates/lanspread-peer/src/network.rs @@ -14,7 +14,11 @@ use s2n_quic::{ Client as QuicClient, Connection, client::Connect, - provider::{congestion_controller, limits::Limits}, + provider::{ + congestion_controller, + io::tokio::{Builder as QuicIoBuilder, Provider as QuicIoProvider}, + limits::Limits, + }, }; use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec}; @@ -23,6 +27,7 @@ use crate::config::{ QUIC_CONNECTION_DATA_WINDOW, QUIC_INITIAL_CONGESTION_WINDOW, QUIC_MAX_SEND_BUFFER_SIZE, + QUIC_SOCKET_BUFFER_SIZE, QUIC_STREAM_DATA_WINDOW, }; @@ -41,13 +46,21 @@ pub(crate) fn quic_congestion_controller() -> congestion_controller::Bbr { .build() } +pub(crate) fn quic_io(addr: SocketAddr) -> eyre::Result { + Ok(QuicIoBuilder::default() + .with_receive_address(addr)? + .with_send_buffer_size(QUIC_SOCKET_BUFFER_SIZE)? + .with_recv_buffer_size(QUIC_SOCKET_BUFFER_SIZE)? + .build()?) +} + /// Establishes a QUIC connection to a peer. pub async fn connect_to_peer(addr: SocketAddr) -> eyre::Result { let limits = quic_limits()?.with_max_handshake_duration(Duration::from_secs(3))?; let client = QuicClient::builder() .with_tls(CERT_PEM)? - .with_io("0.0.0.0:0")? + .with_io(quic_io(SocketAddr::from(([0, 0, 0, 0], 0)))?)? .with_limits(limits)? .with_congestion_controller(quic_congestion_controller())? .start()?; diff --git a/crates/lanspread-peer/src/services/server.rs b/crates/lanspread-peer/src/services/server.rs index c9e636d..4a37d68 100644 --- a/crates/lanspread-peer/src/services/server.rs +++ b/crates/lanspread-peer/src/services/server.rs @@ -10,7 +10,7 @@ use crate::{ config::{CERT_PEM, KEY_PEM}, context::PeerCtx, events, - network::{quic_congestion_controller, quic_limits}, + network::{quic_congestion_controller, quic_io, quic_limits}, services::{ advertise::{monitor_mdns_events, start_mdns_advertiser}, stream::handle_peer_stream, @@ -29,7 +29,7 @@ pub async fn run_server_component( let mut server = Server::builder() .with_tls((CERT_PEM, KEY_PEM))? - .with_io(addr)? + .with_io(quic_io(addr)?)? .with_limits(limits)? .with_congestion_controller(quic_congestion_controller())? .start()?;