From 67c605620374332e443f34aefb00b807e96d6f83 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Tue, 21 Jul 2026 18:27:47 +0200 Subject: [PATCH] feat(cli): move pledged source-size selection to Rust Move the fileio source-size precedence policy into the Rust backend while keeping the private CCtx pledge call, error handling, and diagnostics in the C adapter. A known statted source size, including zero, remains authoritative; when it is unavailable, a positive declared stream size is used, otherwise the zstd unknown-content sentinel is preserved. The Rust policy accepts both sentinels explicitly so the bridge does not couple their representations. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 - ulimit -v 41943040; make -j1 -C tests test --- programs/fileio.c | 24 ++++++++------ rust/src/fileio_backend.rs | 67 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index ad55b137e..7aad30953 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1701,6 +1701,13 @@ int FIO_rust_compressZstdFrame( const char* srcFileName, U64 srcFileSize, int compressionLevel, U64* readsize, U64* compressedSize, size_t* zstdResult); +/* Select the content size to pledge for the next zstd frame. Rust owns the + * source-size precedence policy; C retains the private CCtx call and error + * handling in FIO_rust_compressZstdCallback(). */ +U64 FIO_rust_selectPledgedSrcSize( + U64 srcFileSize, U64 streamSrcSize, + U64 unknownSrcFileSize, U64 unknownPledgedSrcSize); + enum { FIO_RUST_GZIP_OK = 0, FIO_RUST_GZIP_INIT_ERROR = 1, @@ -3128,16 +3135,13 @@ FIO_rust_compressZstdCallback(void* fCtx, void* prefs, void* ress, DISPLAYLEVEL(6, "compression using zstd format \n"); - /* Keep pledged-size and memory diagnostics in C while Rust owns the - * surrounding asynchronous stream loop. */ - if (srcFileSize != UTIL_FILESIZE_UNKNOWN) { - pledgedSrcSize = srcFileSize; - CHECK(ZSTD_CCtx_setPledgedSrcSize(ressPtr->cctx, srcFileSize)); - } else if (prefsPtr->streamSrcSize > 0) { - /* unknown source size; use the declared stream size */ - pledgedSrcSize = prefsPtr->streamSrcSize; - CHECK(ZSTD_CCtx_setPledgedSrcSize(ressPtr->cctx, prefsPtr->streamSrcSize)); - } + /* Rust selects the source-size precedence. C retains the private CCtx + * operation and its exact diagnostic path. */ + pledgedSrcSize = FIO_rust_selectPledgedSrcSize( + srcFileSize, (U64)prefsPtr->streamSrcSize, + UTIL_FILESIZE_UNKNOWN, ZSTD_CONTENTSIZE_UNKNOWN); + if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) + CHECK(ZSTD_CCtx_setPledgedSrcSize(ressPtr->cctx, pledgedSrcSize)); { int windowLog; UTIL_HumanReadableSize_t windowSize; diff --git a/rust/src/fileio_backend.rs b/rust/src/fileio_backend.rs index 37f38c92d..603fa252c 100644 --- a/rust/src/fileio_backend.rs +++ b/rust/src/fileio_backend.rs @@ -1051,6 +1051,46 @@ const FIO_MMAP_DICT: c_int = 1; const FIO_MMAP_POLICY_ENABLE: c_int = 1; const FIO_MMAP_POLICY_DISABLE: c_int = 2; +/// Chooses the source size to pledge for a zstd frame. +/// +/// A statted source size has precedence over the user-declared stream size. +/// The stream size is useful only when the source size is unknown and is +/// positive; otherwise the caller's unknown-content sentinel is preserved. +/// The sentinels are parameters so this policy does not assume that the CLI's +/// file-size marker and zstd's content-size marker are represented by the +/// same constant. +#[inline] +fn select_pledged_src_size( + src_file_size: u64, + stream_src_size: u64, + unknown_src_file_size: u64, + unknown_pledged_src_size: u64, +) -> u64 { + if src_file_size != unknown_src_file_size { + src_file_size + } else if stream_src_size > 0 { + stream_src_size + } else { + unknown_pledged_src_size + } +} + +/// Exposes the CLI source-size precedence policy to its C adapter. +#[no_mangle] +pub extern "C" fn FIO_rust_selectPledgedSrcSize( + src_file_size: u64, + stream_src_size: u64, + unknown_src_file_size: u64, + unknown_pledged_src_size: u64, +) -> u64 { + select_pledged_src_size( + src_file_size, + stream_src_size, + unknown_src_file_size, + unknown_pledged_src_size, + ) +} + /// Selects the dictionary backing store without performing any filesystem I/O. /// /// Explicit mmap enable wins for ordinary dictionaries, patch mode requests a @@ -1170,6 +1210,33 @@ mod tests { ); } + #[test] + fn pledged_source_size_prefers_known_source_then_positive_stream_size() { + let unknown_source = u64::MAX; + let unknown_pledged = u64::MAX - 1; + + assert_eq!( + FIO_rust_selectPledgedSrcSize(123, 456, unknown_source, unknown_pledged), + 123 + ); + assert_eq!( + FIO_rust_selectPledgedSrcSize(0, 456, unknown_source, unknown_pledged), + 0 + ); + assert_eq!( + FIO_rust_selectPledgedSrcSize(unknown_source, 456, unknown_source, unknown_pledged,), + 456 + ); + assert_eq!( + FIO_rust_selectPledgedSrcSize(unknown_source, 0, unknown_source, unknown_pledged,), + unknown_pledged + ); + assert_eq!( + FIO_rust_selectPledgedSrcSize(unknown_source, 456, 123, unknown_pledged), + unknown_pledged + ); + } + fn open_source(path: &Path) -> (c_int, *mut c_void) { let path = c_path(path); let mut stat_buf = MaybeUninit::::uninit();