fix(peer): cap public control and extractor concurrency

Limit server-wide decoded control requests to 16 and gate Stream Install providers behind a separate two-slot semaphore. The provider permit remains held through producer cleanup, bounding concurrent unrar work while ordinary bulk transfers retain their own capacity.

Test Plan:
- just test (passed after rerunning one transient ETXTBSY failure)
- just fmt (Rust formatting passed; repository's generated security report still triggers pre-existing rumdl violations)
- git diff --check
This commit is contained in:
2026-09-12 12:32:12 +02:00
parent 078294ac83
commit ff962801ae
3 changed files with 36 additions and 5 deletions
+13 -2
View File
@@ -35,10 +35,13 @@ const MAX_ESTABLISHED_CONNECTIONS: usize = 64;
/// Mirrors the transport stream limit and bounds application stream futures. /// Mirrors the transport stream limit and bounds application stream futures.
const MAX_CONTROL_STREAM_TASKS: usize = 32; const MAX_CONTROL_STREAM_TASKS: usize = 32;
/// Server-wide cap acquired before any length-delimited decoder is allocated. /// Server-wide cap acquired before any length-delimited decoder is allocated.
const MAX_GLOBAL_CONTROL_STREAM_TASKS: usize = 64; const MAX_GLOBAL_CONTROL_STREAM_TASKS: usize = 16;
/// Long-lived transfers move from the decoder pool to this smaller pool so /// Long-lived transfers move from the decoder pool to this smaller pool so
/// saturated bulk egress cannot consume every control-plane permit. /// saturated bulk egress cannot consume every control-plane permit.
const MAX_GLOBAL_BULK_TRANSFER_TASKS: usize = 48; const MAX_GLOBAL_BULK_TRANSFER_TASKS: usize = 48;
/// Native archive extraction is substantially more expensive than ordinary
/// catalog egress, so keep it behind a separate small global budget.
const MAX_GLOBAL_STREAM_INSTALL_TASKS: usize = 2;
/// Application idle bound for an established connection with no active /// Application idle bound for an established connection with no active
/// request streams. This is independent of transport keepalive traffic. /// request streams. This is independent of transport keepalive traffic.
const CONNECTION_NO_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(10); const CONNECTION_NO_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(10);
@@ -125,6 +128,7 @@ async fn run_server_body(
let mut connection_tasks = JoinSet::new(); let mut connection_tasks = JoinSet::new();
let control_stream_permits = Arc::new(Semaphore::new(MAX_GLOBAL_CONTROL_STREAM_TASKS)); let control_stream_permits = Arc::new(Semaphore::new(MAX_GLOBAL_CONTROL_STREAM_TASKS));
let bulk_transfer_permits = Arc::new(Semaphore::new(MAX_GLOBAL_BULK_TRANSFER_TASKS)); let bulk_transfer_permits = Arc::new(Semaphore::new(MAX_GLOBAL_BULK_TRANSFER_TASKS));
let stream_install_permits = Arc::new(Semaphore::new(MAX_GLOBAL_STREAM_INSTALL_TASKS));
let ready_addr = let ready_addr =
(*ctx.local_peer_addr.read().await).unwrap_or_else(|| direct_connect_addr(server_addr)); (*ctx.local_peer_addr.read().await).unwrap_or_else(|| direct_connect_addr(server_addr));
@@ -174,6 +178,7 @@ async fn run_server_body(
server_children_shutdown.clone(), server_children_shutdown.clone(),
Arc::clone(&control_stream_permits), Arc::clone(&control_stream_permits),
Arc::clone(&bulk_transfer_permits), Arc::clone(&bulk_transfer_permits),
Arc::clone(&stream_install_permits),
)); ));
} }
} }
@@ -265,6 +270,7 @@ async fn handle_peer_connection(
server_shutdown: CancellationToken, server_shutdown: CancellationToken,
control_stream_permits: Arc<Semaphore>, control_stream_permits: Arc<Semaphore>,
bulk_transfer_permits: Arc<Semaphore>, bulk_transfer_permits: Arc<Semaphore>,
stream_install_permits: Arc<Semaphore>,
) -> eyre::Result<()> { ) -> eyre::Result<()> {
let remote_addr = connection.remote_addr()?; let remote_addr = connection.remote_addr()?;
log::info!("{remote_addr} peer connected"); log::info!("{remote_addr} peer connected");
@@ -313,6 +319,7 @@ async fn handle_peer_connection(
stream_shutdown, stream_shutdown,
control_permit, control_permit,
Arc::clone(&bulk_transfer_permits), Arc::clone(&bulk_transfer_permits),
Arc::clone(&stream_install_permits),
)); ));
} }
Ok(None) => break Ok(()), Ok(None) => break Ok(()),
@@ -353,6 +360,7 @@ async fn handle_admitted_peer_stream(
stream_shutdown: CancellationToken, stream_shutdown: CancellationToken,
control_permit: OwnedSemaphorePermit, control_permit: OwnedSemaphorePermit,
bulk_transfer_permits: Arc<Semaphore>, bulk_transfer_permits: Arc<Semaphore>,
stream_install_permits: Arc<Semaphore>,
) -> eyre::Result<()> { ) -> eyre::Result<()> {
handle_peer_stream( handle_peer_stream(
stream, stream,
@@ -361,6 +369,7 @@ async fn handle_admitted_peer_stream(
stream_shutdown, stream_shutdown,
control_permit, control_permit,
bulk_transfer_permits, bulk_transfer_permits,
stream_install_permits,
) )
.await .await
} }
@@ -410,6 +419,7 @@ mod tests {
MAX_ESTABLISHED_CONNECTIONS, MAX_ESTABLISHED_CONNECTIONS,
MAX_GLOBAL_BULK_TRANSFER_TASKS, MAX_GLOBAL_BULK_TRANSFER_TASKS,
MAX_GLOBAL_CONTROL_STREAM_TASKS, MAX_GLOBAL_CONTROL_STREAM_TASKS,
MAX_GLOBAL_STREAM_INSTALL_TASKS,
MAX_INFLIGHT_HANDSHAKES, MAX_INFLIGHT_HANDSHAKES,
bounded_endpoint_limits, bounded_endpoint_limits,
drain_joined_child_tasks, drain_joined_child_tasks,
@@ -423,8 +433,9 @@ mod tests {
assert_eq!(MAX_INFLIGHT_HANDSHAKES, 64); assert_eq!(MAX_INFLIGHT_HANDSHAKES, 64);
assert_eq!(MAX_ESTABLISHED_CONNECTIONS, 64); assert_eq!(MAX_ESTABLISHED_CONNECTIONS, 64);
assert_eq!(MAX_CONTROL_STREAM_TASKS, 32); assert_eq!(MAX_CONTROL_STREAM_TASKS, 32);
assert_eq!(MAX_GLOBAL_CONTROL_STREAM_TASKS, 64); assert_eq!(MAX_GLOBAL_CONTROL_STREAM_TASKS, 16);
assert_eq!(MAX_GLOBAL_BULK_TRANSFER_TASKS, 48); assert_eq!(MAX_GLOBAL_BULK_TRANSFER_TASKS, 48);
assert_eq!(MAX_GLOBAL_STREAM_INSTALL_TASKS, 2);
assert_eq!( assert_eq!(
u64::try_from(MAX_CONTROL_STREAM_TASKS).expect("stream task bound fits u64"), u64::try_from(MAX_CONTROL_STREAM_TASKS).expect("stream task bound fits u64"),
crate::quic_runtime::MAX_OPEN_BIDIRECTIONAL_STREAMS, crate::quic_runtime::MAX_OPEN_BIDIRECTIONAL_STREAMS,
+22 -3
View File
@@ -61,6 +61,7 @@ pub(super) async fn handle_peer_stream(
stream_shutdown: CancellationToken, stream_shutdown: CancellationToken,
control_permit: OwnedSemaphorePermit, control_permit: OwnedSemaphorePermit,
bulk_transfer_permits: Arc<Semaphore>, bulk_transfer_permits: Arc<Semaphore>,
stream_install_permits: Arc<Semaphore>,
) -> eyre::Result<()> { ) -> eyre::Result<()> {
let (rx, tx) = stream.split(); let (rx, tx) = stream.split();
let mut framed_rx = FramedRead::new(rx, request_codec()); let mut framed_rx = FramedRead::new(rx, request_codec());
@@ -94,6 +95,7 @@ pub(super) async fn handle_peer_stream(
source_ip, source_ip,
framed_tx, framed_tx,
&stream_shutdown, &stream_shutdown,
&stream_install_permits,
) )
.await; .await;
framed_tx = dispatched.writer; framed_tx = dispatched.writer;
@@ -111,6 +113,7 @@ pub(super) async fn handle_peer_stream(
source_ip, source_ip,
framed_tx, framed_tx,
&stream_shutdown, &stream_shutdown,
&stream_install_permits,
) )
.await; .await;
framed_tx = dispatched.writer; framed_tx = dispatched.writer;
@@ -229,6 +232,7 @@ async fn dispatch_request(
source_ip: Option<std::net::IpAddr>, source_ip: Option<std::net::IpAddr>,
framed_tx: ResponseWriter, framed_tx: ResponseWriter,
stream_shutdown: &CancellationToken, stream_shutdown: &CancellationToken,
stream_install_permits: &Arc<Semaphore>,
) -> DispatchResult { ) -> DispatchResult {
match request { match request {
Request::Ping => { Request::Ping => {
@@ -322,10 +326,25 @@ async fn dispatch_request(
Request::StreamInstall { Request::StreamInstall {
game_id, game_id,
content_id, content_id,
} => DispatchResult::close( } => {
handle_stream_install_request(ctx, game_id, content_id, framed_tx, stream_shutdown) let Ok(stream_install_permit) = Arc::clone(stream_install_permits).try_acquire_owned()
else {
let mut tx = framed_tx.into_inner();
let _ = tx.reset(application::Error::UNKNOWN);
return DispatchResult::reset(FramedWrite::new(tx, control_codec()));
};
DispatchResult::close(
handle_stream_install_request(
ctx,
game_id,
content_id,
framed_tx,
stream_shutdown,
stream_install_permit,
)
.await, .await,
), )
}
} }
} }
@@ -438,6 +438,7 @@ pub(super) async fn handle_stream_install_request(
content_id: ContentId, content_id: ContentId,
framed_tx: ResponseWriter, framed_tx: ResponseWriter,
stream_shutdown: &CancellationToken, stream_shutdown: &CancellationToken,
_stream_install_permit: tokio::sync::OwnedSemaphorePermit,
) -> ResponseWriter { ) -> ResponseWriter {
log::info!("Received StreamInstall request for {game_id} from peer"); log::info!("Received StreamInstall request for {game_id} from peer");
let mut tx = framed_tx.into_inner(); let mut tx = framed_tx.into_inner();