From cc43ebac4e40a96453f3f8b12eef98d59fee488b Mon Sep 17 00:00:00 2001 From: ddidderr Date: Tue, 21 Jul 2026 20:37:11 +0200 Subject: [PATCH] feat(cli): move multi-file summary policy to Rust The multi-file compression and decompression entry points already delegate file iteration and resource-sensitive work to Rust, but each still kept the final summary decision and progress-to-summary ordering in C. Move that scalar policy into a Rust ABI function. The C side now supplies only exact legacy display callbacks and unchanged format strings, while Rust receives plain counters and a summary kind without exposing FIO, cRess, dRess, or codec layouts. Focused unit tests cover both summary modes, callback order, argument forwarding, and sessions that must not display a summary. Test Plan: - `git diff --check -- programs/fileio.c rust/src/fileio_asyncio.rs` -- passed - `git diff --cached --check` -- passed - `rustfmt --check --edition 2021 rust/src/fileio_asyncio.rs` -- parsed the file; reports pre-existing formatting differences elsewhere in the file, so no unrelated formatting was applied - Cargo, make, and runtime tests were intentionally not run per worker scope --- programs/fileio.c | 117 +++++++++++++++++++----- rust/src/fileio_asyncio.rs | 180 +++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 23 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9bba8dfb9..140a4e832 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -292,6 +292,44 @@ typedef char FIO_rust_compression_params_size[ int FIO_shouldDisplayFileSummary(const FIO_ctx_t* fCtx); int FIO_shouldDisplayMultipleFileSummary(const FIO_ctx_t* fCtx); +enum { + FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION = 0, + FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION = 1 +}; + +typedef void (*FIO_rust_multiple_summary_progress_fn)(void* opaque); +typedef void (*FIO_rust_compress_multiple_summary_fn)( + void* opaque, int nbFilesProcessed, U64 totalBytesInput, U64 totalBytesOutput); +typedef void (*FIO_rust_decompress_multiple_summary_fn)( + void* opaque, int nbFilesProcessed, U64 totalBytesOutput); +typedef struct { + void* opaque; + FIO_rust_multiple_summary_progress_fn displayProgress; + FIO_rust_compress_multiple_summary_fn displayCompression; + FIO_rust_decompress_multiple_summary_fn displayDecompression; +} FIO_rust_multiple_summary_callbacks_t; +typedef char FIO_rust_multiple_summary_callback_sizes[ + (sizeof(FIO_rust_multiple_summary_progress_fn) == sizeof(void*) + && sizeof(FIO_rust_compress_multiple_summary_fn) == sizeof(void*) + && sizeof(FIO_rust_decompress_multiple_summary_fn) == sizeof(void*)) ? 1 : -1]; +typedef char FIO_rust_multiple_summary_opaque_offset[ + (offsetof(FIO_rust_multiple_summary_callbacks_t, opaque) == 0) ? 1 : -1]; +typedef char FIO_rust_multiple_summary_progress_offset[ + (offsetof(FIO_rust_multiple_summary_callbacks_t, displayProgress) + == sizeof(void*)) ? 1 : -1]; +typedef char FIO_rust_multiple_summary_compression_offset[ + (offsetof(FIO_rust_multiple_summary_callbacks_t, displayCompression) + == 2 * sizeof(void*)) ? 1 : -1]; +typedef char FIO_rust_multiple_summary_decompression_offset[ + (offsetof(FIO_rust_multiple_summary_callbacks_t, displayDecompression) + == 3 * sizeof(void*)) ? 1 : -1]; +typedef char FIO_rust_multiple_summary_callbacks_size[ + (sizeof(FIO_rust_multiple_summary_callbacks_t) == 4 * sizeof(void*)) ? 1 : -1]; +void FIO_rust_displayMultipleFileSummary( + int nbFilesTotal, int nbFilesProcessed, + U64 totalBytesInput, U64 totalBytesOutput, int summaryKind, + const FIO_rust_multiple_summary_callbacks_t* callbacks); + typedef void (*FIO_rust_multi_files_concat_fatal_fn)(void* opaque); typedef void (*FIO_rust_multi_files_concat_warning_fn)(void* opaque, int hasStdoutOutput, @@ -1297,6 +1335,58 @@ static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs outFileName, &callbacks); } +/* Rust owns the final multi-file summary decision and ordering. These + * callbacks retain the exact C display sinks and format strings. */ +static void FIO_rust_multipleSummaryDisplayProgress(void* opaque) +{ + (void)opaque; + DISPLAY_PROGRESS("\r%79s\r", ""); +} + +static void FIO_rust_compressMultipleSummary(void* opaque, int nbFilesProcessed, + U64 totalBytesInput, U64 totalBytesOutput) +{ + UTIL_HumanReadableSize_t const hr_isize = + UTIL_makeHumanReadableSize(totalBytesInput); + UTIL_HumanReadableSize_t const hr_osize = + UTIL_makeHumanReadableSize(totalBytesOutput); + (void)opaque; + + if (totalBytesInput == 0) { + DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n", + nbFilesProcessed, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix); + } else { + DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n", + nbFilesProcessed, + (double)totalBytesOutput/(double)totalBytesInput*100, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix); + } +} + +static void FIO_rust_decompressMultipleSummary(void* opaque, int nbFilesProcessed, + U64 totalBytesOutput) +{ + (void)opaque; + DISPLAY_SUMMARY("%d files decompressed : %6llu bytes total \n", + nbFilesProcessed, (unsigned long long)totalBytesOutput); +} + +static void FIO_displayMultipleFileSummary(const FIO_ctx_t* fCtx, int summaryKind) +{ + FIO_rust_multiple_summary_callbacks_t callbacks; + callbacks.opaque = NULL; + callbacks.displayProgress = FIO_rust_multipleSummaryDisplayProgress; + callbacks.displayCompression = FIO_rust_compressMultipleSummary; + callbacks.displayDecompression = FIO_rust_decompressMultipleSummary; + FIO_rust_displayMultipleFileSummary( + fCtx->nbFilesTotal, fCtx->nbFilesProcessed, + (U64)fCtx->totalBytesInput, (U64)fCtx->totalBytesOutput, + summaryKind, &callbacks); +} + #ifndef ZSTD_NOCOMPRESS /* ********************************************************************** @@ -3913,24 +4003,8 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal); } - if (FIO_shouldDisplayMultipleFileSummary(fCtx)) { - UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput); - UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); - - DISPLAY_PROGRESS("\r%79s\r", ""); - if (fCtx->totalBytesInput == 0) { - DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n", - fCtx->nbFilesProcessed, - hr_isize.precision, hr_isize.value, hr_isize.suffix, - hr_osize.precision, hr_osize.value, hr_osize.suffix); - } else { - DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n", - fCtx->nbFilesProcessed, - (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, - hr_isize.precision, hr_isize.value, hr_isize.suffix, - hr_osize.precision, hr_osize.value, hr_osize.suffix); - } - } + FIO_displayMultipleFileSummary( + fCtx, FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION); FIO_freeCResources(&ress); return error; @@ -5185,11 +5259,8 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal); } - if (FIO_shouldDisplayMultipleFileSummary(fCtx)) { - DISPLAY_PROGRESS("\r%79s\r", ""); - DISPLAY_SUMMARY("%d files decompressed : %6llu bytes total \n", - fCtx->nbFilesProcessed, (unsigned long long)fCtx->totalBytesOutput); - } + FIO_displayMultipleFileSummary( + fCtx, FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION); FIO_freeDResources(ress); return error; diff --git a/rust/src/fileio_asyncio.rs b/rust/src/fileio_asyncio.rs index 9de3bc86f..9174b8e76 100644 --- a/rust/src/fileio_asyncio.rs +++ b/rust/src/fileio_asyncio.rs @@ -1598,6 +1598,73 @@ struct FIO_rust_compression_context_t { totalBytesOutput: usize, } +pub const FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION: c_int = 0; +pub const FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION: c_int = 1; + +type FIO_rust_multiple_summary_progress_fn = unsafe extern "C" fn(*mut c_void); +type FIO_rust_compress_multiple_summary_fn = + unsafe extern "C" fn(*mut c_void, c_int, u64, u64); +type FIO_rust_decompress_multiple_summary_fn = unsafe extern "C" fn(*mut c_void, c_int, u64); + +/// C supplies only display sinks. The multi-file counters are passed as +/// scalars so the summary policy never depends on private FIO/cRess/dRess +/// layouts or external codec state. +#[repr(C)] +pub struct FIO_rust_multiple_summary_callbacks_t { + pub opaque: *mut c_void, + pub display_progress: Option, + pub display_compression: Option, + pub display_decompression: Option, +} + +/// Runs the final multi-file summary in the same position and order as the +/// original C callers: clear progress first, then select the compression or +/// decompression summary sink. C retains only the exact display formatting +/// callbacks, preserving diagnostics and the CLI ABI. +#[no_mangle] +pub unsafe extern "C" fn FIO_rust_displayMultipleFileSummary( + nb_files_total: c_int, + nb_files_processed: c_int, + total_bytes_input: u64, + total_bytes_output: u64, + summary_kind: c_int, + callbacks: *const FIO_rust_multiple_summary_callbacks_t, +) { + if nb_files_total <= 1 || nb_files_processed < 1 { + return; + } + + assert!(!callbacks.is_null()); + let callbacks = unsafe { &*callbacks }; + let display_progress = callbacks + .display_progress + .expect("multi-file summary progress callback is required"); + unsafe { display_progress(callbacks.opaque) }; + + match summary_kind { + FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION => { + let display = callbacks + .display_compression + .expect("compression summary callback is required"); + unsafe { + display( + callbacks.opaque, + nb_files_processed, + total_bytes_input, + total_bytes_output, + ) + }; + } + FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION => { + let display = callbacks + .display_decompression + .expect("decompression summary callback is required"); + unsafe { display(callbacks.opaque, nb_files_processed, total_bytes_output) }; + } + _ => unreachable!("invalid multi-file summary kind"), + } +} + type PoolFunction = unsafe extern "C" fn(*mut c_void); #[cfg(not(test))] @@ -7328,6 +7395,119 @@ mod tests { statuses: Vec, } + #[derive(Default)] + struct MultipleSummaryState { + events: Vec<&'static str>, + compression: Option<(c_int, u64, u64)>, + decompression: Option<(c_int, u64)>, + } + + unsafe extern "C" fn record_multiple_summary_progress(opaque: *mut c_void) { + let state = unsafe { &mut *opaque.cast::() }; + state.events.push("progress"); + } + + unsafe extern "C" fn record_multiple_compression_summary( + opaque: *mut c_void, + nb_files_processed: c_int, + total_bytes_input: u64, + total_bytes_output: u64, + ) { + let state = unsafe { &mut *opaque.cast::() }; + state.events.push("compression"); + state.compression = Some(( + nb_files_processed, + total_bytes_input, + total_bytes_output, + )); + } + + unsafe extern "C" fn record_multiple_decompression_summary( + opaque: *mut c_void, + nb_files_processed: c_int, + total_bytes_output: u64, + ) { + let state = unsafe { &mut *opaque.cast::() }; + state.events.push("decompression"); + state.decompression = Some((nb_files_processed, total_bytes_output)); + } + + fn multiple_summary_callbacks( + state: &mut MultipleSummaryState, + ) -> FIO_rust_multiple_summary_callbacks_t { + FIO_rust_multiple_summary_callbacks_t { + opaque: (state as *mut MultipleSummaryState).cast(), + display_progress: Some(record_multiple_summary_progress), + display_compression: Some(record_multiple_compression_summary), + display_decompression: Some(record_multiple_decompression_summary), + } + } + + #[test] + fn multiple_file_summary_dispatches_compression_after_progress() { + let mut state = MultipleSummaryState::default(); + let callbacks = multiple_summary_callbacks(&mut state); + + unsafe { + FIO_rust_displayMultipleFileSummary( + 3, + 2, + 4096, + 1024, + FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION, + &callbacks, + ) + }; + + assert_eq!(state.events, ["progress", "compression"]); + assert_eq!(state.compression, Some((2, 4096, 1024))); + assert_eq!(state.decompression, None); + } + + #[test] + fn multiple_file_summary_dispatches_decompression_and_skips_empty_sessions() { + let mut state = MultipleSummaryState::default(); + let callbacks = multiple_summary_callbacks(&mut state); + + unsafe { + FIO_rust_displayMultipleFileSummary( + 4, + 3, + 0, + 777, + FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION, + &callbacks, + ) + }; + assert_eq!(state.events, ["progress", "decompression"]); + assert_eq!(state.decompression, Some((3, 777))); + + state.events.clear(); + state.decompression = None; + unsafe { + FIO_rust_displayMultipleFileSummary( + 1, + 3, + 10, + 20, + FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION, + &callbacks, + ) + }; + unsafe { + FIO_rust_displayMultipleFileSummary( + 4, + 0, + 10, + 20, + FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION, + &callbacks, + ) + }; + assert!(state.events.is_empty()); + assert_eq!(state.decompression, None); + } + unsafe extern "C" fn record_multiple_compression_file( opaque: *mut c_void, destination: *const c_char,