feat(cli): move multi-file summary policy to Rust
The multi-file compression and decompression entry points already delegate file iteration and resource-sensitive work to Rust, but each still kept the final summary decision and progress-to-summary ordering in C. Move that scalar policy into a Rust ABI function. The C side now supplies only exact legacy display callbacks and unchanged format strings, while Rust receives plain counters and a summary kind without exposing FIO, cRess, dRess, or codec layouts. Focused unit tests cover both summary modes, callback order, argument forwarding, and sessions that must not display a summary. Test Plan: - `git diff --check -- programs/fileio.c rust/src/fileio_asyncio.rs` -- passed - `git diff --cached --check` -- passed - `rustfmt --check --edition 2021 rust/src/fileio_asyncio.rs` -- parsed the file; reports pre-existing formatting differences elsewhere in the file, so no unrelated formatting was applied - Cargo, make, and runtime tests were intentionally not run per worker scope
This commit is contained in:
+94
-23
@@ -292,6 +292,44 @@ typedef char FIO_rust_compression_params_size[
|
|||||||
int FIO_shouldDisplayFileSummary(const FIO_ctx_t* fCtx);
|
int FIO_shouldDisplayFileSummary(const FIO_ctx_t* fCtx);
|
||||||
int FIO_shouldDisplayMultipleFileSummary(const FIO_ctx_t* fCtx);
|
int FIO_shouldDisplayMultipleFileSummary(const FIO_ctx_t* fCtx);
|
||||||
|
|
||||||
|
enum {
|
||||||
|
FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION = 0,
|
||||||
|
FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION = 1
|
||||||
|
};
|
||||||
|
|
||||||
|
typedef void (*FIO_rust_multiple_summary_progress_fn)(void* opaque);
|
||||||
|
typedef void (*FIO_rust_compress_multiple_summary_fn)(
|
||||||
|
void* opaque, int nbFilesProcessed, U64 totalBytesInput, U64 totalBytesOutput);
|
||||||
|
typedef void (*FIO_rust_decompress_multiple_summary_fn)(
|
||||||
|
void* opaque, int nbFilesProcessed, U64 totalBytesOutput);
|
||||||
|
typedef struct {
|
||||||
|
void* opaque;
|
||||||
|
FIO_rust_multiple_summary_progress_fn displayProgress;
|
||||||
|
FIO_rust_compress_multiple_summary_fn displayCompression;
|
||||||
|
FIO_rust_decompress_multiple_summary_fn displayDecompression;
|
||||||
|
} FIO_rust_multiple_summary_callbacks_t;
|
||||||
|
typedef char FIO_rust_multiple_summary_callback_sizes[
|
||||||
|
(sizeof(FIO_rust_multiple_summary_progress_fn) == sizeof(void*)
|
||||||
|
&& sizeof(FIO_rust_compress_multiple_summary_fn) == sizeof(void*)
|
||||||
|
&& sizeof(FIO_rust_decompress_multiple_summary_fn) == sizeof(void*)) ? 1 : -1];
|
||||||
|
typedef char FIO_rust_multiple_summary_opaque_offset[
|
||||||
|
(offsetof(FIO_rust_multiple_summary_callbacks_t, opaque) == 0) ? 1 : -1];
|
||||||
|
typedef char FIO_rust_multiple_summary_progress_offset[
|
||||||
|
(offsetof(FIO_rust_multiple_summary_callbacks_t, displayProgress)
|
||||||
|
== sizeof(void*)) ? 1 : -1];
|
||||||
|
typedef char FIO_rust_multiple_summary_compression_offset[
|
||||||
|
(offsetof(FIO_rust_multiple_summary_callbacks_t, displayCompression)
|
||||||
|
== 2 * sizeof(void*)) ? 1 : -1];
|
||||||
|
typedef char FIO_rust_multiple_summary_decompression_offset[
|
||||||
|
(offsetof(FIO_rust_multiple_summary_callbacks_t, displayDecompression)
|
||||||
|
== 3 * sizeof(void*)) ? 1 : -1];
|
||||||
|
typedef char FIO_rust_multiple_summary_callbacks_size[
|
||||||
|
(sizeof(FIO_rust_multiple_summary_callbacks_t) == 4 * sizeof(void*)) ? 1 : -1];
|
||||||
|
void FIO_rust_displayMultipleFileSummary(
|
||||||
|
int nbFilesTotal, int nbFilesProcessed,
|
||||||
|
U64 totalBytesInput, U64 totalBytesOutput, int summaryKind,
|
||||||
|
const FIO_rust_multiple_summary_callbacks_t* callbacks);
|
||||||
|
|
||||||
typedef void (*FIO_rust_multi_files_concat_fatal_fn)(void* opaque);
|
typedef void (*FIO_rust_multi_files_concat_fatal_fn)(void* opaque);
|
||||||
typedef void (*FIO_rust_multi_files_concat_warning_fn)(void* opaque,
|
typedef void (*FIO_rust_multi_files_concat_warning_fn)(void* opaque,
|
||||||
int hasStdoutOutput,
|
int hasStdoutOutput,
|
||||||
@@ -1297,6 +1335,58 @@ static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs
|
|||||||
outFileName, &callbacks);
|
outFileName, &callbacks);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Rust owns the final multi-file summary decision and ordering. These
|
||||||
|
* callbacks retain the exact C display sinks and format strings. */
|
||||||
|
static void FIO_rust_multipleSummaryDisplayProgress(void* opaque)
|
||||||
|
{
|
||||||
|
(void)opaque;
|
||||||
|
DISPLAY_PROGRESS("\r%79s\r", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FIO_rust_compressMultipleSummary(void* opaque, int nbFilesProcessed,
|
||||||
|
U64 totalBytesInput, U64 totalBytesOutput)
|
||||||
|
{
|
||||||
|
UTIL_HumanReadableSize_t const hr_isize =
|
||||||
|
UTIL_makeHumanReadableSize(totalBytesInput);
|
||||||
|
UTIL_HumanReadableSize_t const hr_osize =
|
||||||
|
UTIL_makeHumanReadableSize(totalBytesOutput);
|
||||||
|
(void)opaque;
|
||||||
|
|
||||||
|
if (totalBytesInput == 0) {
|
||||||
|
DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
|
||||||
|
nbFilesProcessed,
|
||||||
|
hr_isize.precision, hr_isize.value, hr_isize.suffix,
|
||||||
|
hr_osize.precision, hr_osize.value, hr_osize.suffix);
|
||||||
|
} else {
|
||||||
|
DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
|
||||||
|
nbFilesProcessed,
|
||||||
|
(double)totalBytesOutput/(double)totalBytesInput*100,
|
||||||
|
hr_isize.precision, hr_isize.value, hr_isize.suffix,
|
||||||
|
hr_osize.precision, hr_osize.value, hr_osize.suffix);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FIO_rust_decompressMultipleSummary(void* opaque, int nbFilesProcessed,
|
||||||
|
U64 totalBytesOutput)
|
||||||
|
{
|
||||||
|
(void)opaque;
|
||||||
|
DISPLAY_SUMMARY("%d files decompressed : %6llu bytes total \n",
|
||||||
|
nbFilesProcessed, (unsigned long long)totalBytesOutput);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FIO_displayMultipleFileSummary(const FIO_ctx_t* fCtx, int summaryKind)
|
||||||
|
{
|
||||||
|
FIO_rust_multiple_summary_callbacks_t callbacks;
|
||||||
|
callbacks.opaque = NULL;
|
||||||
|
callbacks.displayProgress = FIO_rust_multipleSummaryDisplayProgress;
|
||||||
|
callbacks.displayCompression = FIO_rust_compressMultipleSummary;
|
||||||
|
callbacks.displayDecompression = FIO_rust_decompressMultipleSummary;
|
||||||
|
FIO_rust_displayMultipleFileSummary(
|
||||||
|
fCtx->nbFilesTotal, fCtx->nbFilesProcessed,
|
||||||
|
(U64)fCtx->totalBytesInput, (U64)fCtx->totalBytesOutput,
|
||||||
|
summaryKind, &callbacks);
|
||||||
|
}
|
||||||
|
|
||||||
#ifndef ZSTD_NOCOMPRESS
|
#ifndef ZSTD_NOCOMPRESS
|
||||||
|
|
||||||
/* **********************************************************************
|
/* **********************************************************************
|
||||||
@@ -3913,24 +4003,8 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
|
|||||||
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
|
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
|
FIO_displayMultipleFileSummary(
|
||||||
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput);
|
fCtx, FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION);
|
||||||
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput);
|
|
||||||
|
|
||||||
DISPLAY_PROGRESS("\r%79s\r", "");
|
|
||||||
if (fCtx->totalBytesInput == 0) {
|
|
||||||
DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
|
|
||||||
fCtx->nbFilesProcessed,
|
|
||||||
hr_isize.precision, hr_isize.value, hr_isize.suffix,
|
|
||||||
hr_osize.precision, hr_osize.value, hr_osize.suffix);
|
|
||||||
} else {
|
|
||||||
DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
|
|
||||||
fCtx->nbFilesProcessed,
|
|
||||||
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
|
|
||||||
hr_isize.precision, hr_isize.value, hr_isize.suffix,
|
|
||||||
hr_osize.precision, hr_osize.value, hr_osize.suffix);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
FIO_freeCResources(&ress);
|
FIO_freeCResources(&ress);
|
||||||
return error;
|
return error;
|
||||||
@@ -5185,11 +5259,8 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
|
|||||||
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
|
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
|
FIO_displayMultipleFileSummary(
|
||||||
DISPLAY_PROGRESS("\r%79s\r", "");
|
fCtx, FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION);
|
||||||
DISPLAY_SUMMARY("%d files decompressed : %6llu bytes total \n",
|
|
||||||
fCtx->nbFilesProcessed, (unsigned long long)fCtx->totalBytesOutput);
|
|
||||||
}
|
|
||||||
|
|
||||||
FIO_freeDResources(ress);
|
FIO_freeDResources(ress);
|
||||||
return error;
|
return error;
|
||||||
|
|||||||
@@ -1598,6 +1598,73 @@ struct FIO_rust_compression_context_t {
|
|||||||
totalBytesOutput: usize,
|
totalBytesOutput: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub const FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION: c_int = 0;
|
||||||
|
pub const FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION: c_int = 1;
|
||||||
|
|
||||||
|
type FIO_rust_multiple_summary_progress_fn = unsafe extern "C" fn(*mut c_void);
|
||||||
|
type FIO_rust_compress_multiple_summary_fn =
|
||||||
|
unsafe extern "C" fn(*mut c_void, c_int, u64, u64);
|
||||||
|
type FIO_rust_decompress_multiple_summary_fn = unsafe extern "C" fn(*mut c_void, c_int, u64);
|
||||||
|
|
||||||
|
/// C supplies only display sinks. The multi-file counters are passed as
|
||||||
|
/// scalars so the summary policy never depends on private FIO/cRess/dRess
|
||||||
|
/// layouts or external codec state.
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct FIO_rust_multiple_summary_callbacks_t {
|
||||||
|
pub opaque: *mut c_void,
|
||||||
|
pub display_progress: Option<FIO_rust_multiple_summary_progress_fn>,
|
||||||
|
pub display_compression: Option<FIO_rust_compress_multiple_summary_fn>,
|
||||||
|
pub display_decompression: Option<FIO_rust_decompress_multiple_summary_fn>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs the final multi-file summary in the same position and order as the
|
||||||
|
/// original C callers: clear progress first, then select the compression or
|
||||||
|
/// decompression summary sink. C retains only the exact display formatting
|
||||||
|
/// callbacks, preserving diagnostics and the CLI ABI.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn FIO_rust_displayMultipleFileSummary(
|
||||||
|
nb_files_total: c_int,
|
||||||
|
nb_files_processed: c_int,
|
||||||
|
total_bytes_input: u64,
|
||||||
|
total_bytes_output: u64,
|
||||||
|
summary_kind: c_int,
|
||||||
|
callbacks: *const FIO_rust_multiple_summary_callbacks_t,
|
||||||
|
) {
|
||||||
|
if nb_files_total <= 1 || nb_files_processed < 1 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(!callbacks.is_null());
|
||||||
|
let callbacks = unsafe { &*callbacks };
|
||||||
|
let display_progress = callbacks
|
||||||
|
.display_progress
|
||||||
|
.expect("multi-file summary progress callback is required");
|
||||||
|
unsafe { display_progress(callbacks.opaque) };
|
||||||
|
|
||||||
|
match summary_kind {
|
||||||
|
FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION => {
|
||||||
|
let display = callbacks
|
||||||
|
.display_compression
|
||||||
|
.expect("compression summary callback is required");
|
||||||
|
unsafe {
|
||||||
|
display(
|
||||||
|
callbacks.opaque,
|
||||||
|
nb_files_processed,
|
||||||
|
total_bytes_input,
|
||||||
|
total_bytes_output,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION => {
|
||||||
|
let display = callbacks
|
||||||
|
.display_decompression
|
||||||
|
.expect("decompression summary callback is required");
|
||||||
|
unsafe { display(callbacks.opaque, nb_files_processed, total_bytes_output) };
|
||||||
|
}
|
||||||
|
_ => unreachable!("invalid multi-file summary kind"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type PoolFunction = unsafe extern "C" fn(*mut c_void);
|
type PoolFunction = unsafe extern "C" fn(*mut c_void);
|
||||||
|
|
||||||
#[cfg(not(test))]
|
#[cfg(not(test))]
|
||||||
@@ -7328,6 +7395,119 @@ mod tests {
|
|||||||
statuses: Vec<c_int>,
|
statuses: Vec<c_int>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct MultipleSummaryState {
|
||||||
|
events: Vec<&'static str>,
|
||||||
|
compression: Option<(c_int, u64, u64)>,
|
||||||
|
decompression: Option<(c_int, u64)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn record_multiple_summary_progress(opaque: *mut c_void) {
|
||||||
|
let state = unsafe { &mut *opaque.cast::<MultipleSummaryState>() };
|
||||||
|
state.events.push("progress");
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn record_multiple_compression_summary(
|
||||||
|
opaque: *mut c_void,
|
||||||
|
nb_files_processed: c_int,
|
||||||
|
total_bytes_input: u64,
|
||||||
|
total_bytes_output: u64,
|
||||||
|
) {
|
||||||
|
let state = unsafe { &mut *opaque.cast::<MultipleSummaryState>() };
|
||||||
|
state.events.push("compression");
|
||||||
|
state.compression = Some((
|
||||||
|
nb_files_processed,
|
||||||
|
total_bytes_input,
|
||||||
|
total_bytes_output,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn record_multiple_decompression_summary(
|
||||||
|
opaque: *mut c_void,
|
||||||
|
nb_files_processed: c_int,
|
||||||
|
total_bytes_output: u64,
|
||||||
|
) {
|
||||||
|
let state = unsafe { &mut *opaque.cast::<MultipleSummaryState>() };
|
||||||
|
state.events.push("decompression");
|
||||||
|
state.decompression = Some((nb_files_processed, total_bytes_output));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn multiple_summary_callbacks(
|
||||||
|
state: &mut MultipleSummaryState,
|
||||||
|
) -> FIO_rust_multiple_summary_callbacks_t {
|
||||||
|
FIO_rust_multiple_summary_callbacks_t {
|
||||||
|
opaque: (state as *mut MultipleSummaryState).cast(),
|
||||||
|
display_progress: Some(record_multiple_summary_progress),
|
||||||
|
display_compression: Some(record_multiple_compression_summary),
|
||||||
|
display_decompression: Some(record_multiple_decompression_summary),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn multiple_file_summary_dispatches_compression_after_progress() {
|
||||||
|
let mut state = MultipleSummaryState::default();
|
||||||
|
let callbacks = multiple_summary_callbacks(&mut state);
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
FIO_rust_displayMultipleFileSummary(
|
||||||
|
3,
|
||||||
|
2,
|
||||||
|
4096,
|
||||||
|
1024,
|
||||||
|
FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION,
|
||||||
|
&callbacks,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(state.events, ["progress", "compression"]);
|
||||||
|
assert_eq!(state.compression, Some((2, 4096, 1024)));
|
||||||
|
assert_eq!(state.decompression, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn multiple_file_summary_dispatches_decompression_and_skips_empty_sessions() {
|
||||||
|
let mut state = MultipleSummaryState::default();
|
||||||
|
let callbacks = multiple_summary_callbacks(&mut state);
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
FIO_rust_displayMultipleFileSummary(
|
||||||
|
4,
|
||||||
|
3,
|
||||||
|
0,
|
||||||
|
777,
|
||||||
|
FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION,
|
||||||
|
&callbacks,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert_eq!(state.events, ["progress", "decompression"]);
|
||||||
|
assert_eq!(state.decompression, Some((3, 777)));
|
||||||
|
|
||||||
|
state.events.clear();
|
||||||
|
state.decompression = None;
|
||||||
|
unsafe {
|
||||||
|
FIO_rust_displayMultipleFileSummary(
|
||||||
|
1,
|
||||||
|
3,
|
||||||
|
10,
|
||||||
|
20,
|
||||||
|
FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION,
|
||||||
|
&callbacks,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
unsafe {
|
||||||
|
FIO_rust_displayMultipleFileSummary(
|
||||||
|
4,
|
||||||
|
0,
|
||||||
|
10,
|
||||||
|
20,
|
||||||
|
FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION,
|
||||||
|
&callbacks,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert!(state.events.is_empty());
|
||||||
|
assert_eq!(state.decompression, None);
|
||||||
|
}
|
||||||
|
|
||||||
unsafe extern "C" fn record_multiple_compression_file(
|
unsafe extern "C" fn record_multiple_compression_file(
|
||||||
opaque: *mut c_void,
|
opaque: *mut c_void,
|
||||||
destination: *const c_char,
|
destination: *const c_char,
|
||||||
|
|||||||
Reference in New Issue
Block a user