ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move list formatting to Rust

Browse Source 
Move the per-file and multi-file --list row formatters out of programs/fileio.c. C still owns filesystem access, frame analysis, private fileInfo_t storage, and exact status diagnostics; Rust now formats the rows into byte buffers and sends them through a synchronous output callback. This keeps filenames byte-preserving and prevents the private C record from crossing the ABI while preserving the original display thresholds, human-readable size policy, unavailable-content spacing, checksum order, and total-row behavior.

Replace the old display callbacks with one size-delimited output callback shared by the single-file and multi-file projections. Add explicit Rust ABI layout checks, formatter fixtures for normal/unavailable/verbose/checksum/total output, and callback-order assertions for the policy boundary.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets (208 passed)
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test (all original tests completed successfully)
- git diff --check
This commit is contained in:
ddidderr
2026-07-21 15:24:47 +02:00
parent c42f69e756
commit 13a271c420
2 changed files with 331 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/fileio_prefs.rs
+319 -67
View File
@@ -11,6 +11,7 @@
//! unspecified are not zero-filled here either.
use std::ffi::{c_void, CStr};
use std::fmt::Write as _;
use std::io::{self, Write};
use std::mem::{offset_of, size_of};
use std::os::raw::{c_char, c_int, c_uint};
@@ -256,7 +257,7 @@ pub struct FIO_fileInfo_t {
}
/// The subset of C's private `fileInfo_t` needed by the `--list` aggregate
/// row. C keeps opening, parsing, and formatting each file; Rust owns the
/// row. C keeps opening and parsing each file; Rust owns formatting and the
/// per-file and multi-file policy around those callbacks.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
@@ -890,27 +891,25 @@ pub type FIO_listMultipleFilesListFileFn = unsafe extern "C" fn(
c_int,
*mut FIO_listFileInfoProjection_t,
) -> c_int;
pub type FIO_listMultipleFilesDisplayTotalFn =
unsafe extern "C" fn(*mut c_void, *const FIO_listFileInfoProjection_t);
pub type FIO_listFileGetInfoFn =
unsafe extern "C" fn(*mut c_void, *const c_char, *mut FIO_fileInfo_t) -> c_int;
pub type FIO_listFileDisplayStatusFn =
unsafe extern "C" fn(*mut c_void, c_int, *const c_char, c_int);
pub type FIO_listFileDisplayInfoFn =
unsafe extern "C" fn(*mut c_void, *const c_char, *const FIO_fileInfo_t, c_int);
pub type FIO_listFileWriteOutputFn = unsafe extern "C" fn(*mut c_void, *const c_char, usize);
/// Callbacks for the C-owned `--list` file operation.
///
/// The callbacks keep stat/open, frame analysis, exact status diagnostics, and
/// row formatting in C. Rust only orders those operations and publishes the
/// small aggregate projection consumed by the surrounding multi-file policy.
/// The callbacks keep stat/open, frame analysis, and exact status diagnostics
/// in C. Rust owns row formatting and writes the resulting bytes through the
/// synchronous output callback, then publishes the small aggregate projection
/// consumed by the surrounding multi-file policy.
#[repr(C)]
pub struct FIO_listFileCallbacks_t {
opaque: *mut c_void,
getInfo: Option<FIO_listFileGetInfoFn>,
displayStatus: Option<FIO_listFileDisplayStatusFn>,
displayInfo: Option<FIO_listFileDisplayInfoFn>,
writeOutput: Option<FIO_listFileWriteOutputFn>,
}
#[inline]
@@ -927,8 +926,8 @@ fn list_file_status_action(status: c_int) -> c_int {
/// Map the C-owned `InfoError` result to the scalar `--list` action.
///
/// C retains the exact diagnostics, file opening/parsing, metadata formatting,
/// and private `fileInfo_t` layout. Rust owns only this status policy.
/// C retains the exact diagnostics, file opening/parsing, and private
/// `fileInfo_t` layout. Rust owns the status policy and metadata formatting.
#[no_mangle]
pub extern "C" fn FIO_rust_listFileStatusAction(status: c_int) -> c_int {
list_file_status_action(status)
@@ -947,10 +946,163 @@ fn project_list_file_info(info: &FIO_fileInfo_t) -> FIO_listFileInfoProjection_t
}
}
#[inline]
fn append_fmt(output: &mut Vec<u8>, args: std::fmt::Arguments<'_>) {
let mut text = String::new();
text.write_fmt(args)
.expect("formatting into String cannot fail");
output.extend_from_slice(text.as_bytes());
}
#[inline]
fn append_padded_bytes(output: &mut Vec<u8>, value: &[u8], width: usize) {
output.extend(std::iter::repeat_n(b' ', width.saturating_sub(value.len())));
output.extend_from_slice(value);
}
#[inline]
fn append_integer<T: std::fmt::Display>(output: &mut Vec<u8>, value: T, width: usize) {
let mut text = String::new();
write!(&mut text, "{value}").expect("formatting into String cannot fail");
append_padded_bytes(output, text.as_bytes(), width);
}
#[inline]
fn append_human_size(output: &mut Vec<u8>, size: u64, width: usize) {
let size = crate::util::UTIL_makeHumanReadableSize(size);
let mut number = String::new();
write!(
&mut number,
"{:.*}",
size.precision.max(0) as usize,
size.value
)
.expect("formatting into String cannot fail");
append_padded_bytes(output, number.as_bytes(), width);
let suffix = unsafe { CStr::from_ptr(size.suffix).to_bytes() };
append_padded_bytes(output, suffix, if width == 0 { 0 } else { 4 });
}
#[inline]
fn append_c_string(output: &mut Vec<u8>, value: *const c_char) {
output.extend_from_slice(unsafe { CStr::from_ptr(value).to_bytes() });
}
fn format_list_file_info(
input_name: *const c_char,
info: &FIO_fileInfo_t,
display_level: c_int,
) -> Vec<u8> {
let total_frames = info.numSkippableFrames.wrapping_add(info.numActualFrames);
let ratio = if info.compressedSize == 0 {
0.0
} else {
info.decompressedSize as f64 / info.compressedSize as f64
};
let check = if info.usesCheck != 0 {
b"XXH64".as_slice()
} else {
b"None".as_slice()
};
let mut output = Vec::new();
if display_level <= 2 {
append_integer(&mut output, total_frames, 6);
output.extend_from_slice(b" ");
append_integer(&mut output, info.numSkippableFrames, 5);
output.extend_from_slice(b" ");
append_human_size(&mut output, info.compressedSize, 6);
if info.decompUnavailable == 0 {
output.extend_from_slice(b" ");
append_human_size(&mut output, info.decompressedSize, 8);
output.extend_from_slice(b" ");
append_fmt(&mut output, format_args!("{:>5.3}", ratio));
output.extend_from_slice(b" ");
} else {
output.extend_from_slice(b" ");
}
append_padded_bytes(&mut output, check, 5);
output.extend_from_slice(b" ");
append_c_string(&mut output, input_name);
output.push(b'\n');
return output;
}
append_c_string(&mut output, input_name);
output.extend_from_slice(b" \n# Zstandard Frames: ");
append_integer(&mut output, info.numActualFrames, 0);
output.push(b'\n');
if info.numSkippableFrames != 0 {
output.extend_from_slice(b"# Skippable Frames: ");
append_integer(&mut output, info.numSkippableFrames, 0);
output.push(b'\n');
}
output.extend_from_slice(b"DictID: ");
append_integer(&mut output, info.dictID, 0);
output.extend_from_slice(b"\nWindow Size: ");
append_human_size(&mut output, info.windowSize, 0);
append_fmt(&mut output, format_args!(" ({} B)\n", info.windowSize));
output.extend_from_slice(b"Compressed Size: ");
append_human_size(&mut output, info.compressedSize, 0);
append_fmt(&mut output, format_args!(" ({} B)\n", info.compressedSize));
if info.decompUnavailable == 0 {
output.extend_from_slice(b"Decompressed Size: ");
append_human_size(&mut output, info.decompressedSize, 0);
append_fmt(
&mut output,
format_args!(" ({} B)\nRatio: {:.4}\n", info.decompressedSize, ratio),
);
}
output.extend_from_slice(b"Check: ");
output.extend_from_slice(check);
if info.usesCheck != 0 && info.numActualFrames == 1 {
output.push(b' ');
for byte in info.checksum.iter().rev() {
append_fmt(&mut output, format_args!("{byte:02x}"));
}
}
output.extend_from_slice(b"\n\n");
output
}
fn format_list_total(total: &FIO_listFileInfoProjection_t) -> Vec<u8> {
let total_frames = total.numSkippableFrames.wrapping_add(total.numActualFrames);
let ratio = if total.compressedSize == 0 {
0.0
} else {
total.decompressedSize as f64 / total.compressedSize as f64
};
let check = if total.usesCheck != 0 {
b"XXH64".as_slice()
} else {
b"".as_slice()
};
let mut output =
b"----------------------------------------------------------------- \n".to_vec();
append_integer(&mut output, total_frames, 6);
output.extend_from_slice(b" ");
append_integer(&mut output, total.numSkippableFrames, 5);
output.extend_from_slice(b" ");
append_human_size(&mut output, total.compressedSize, 6);
if total.decompUnavailable == 0 {
output.extend_from_slice(b" ");
append_human_size(&mut output, total.decompressedSize, 8);
output.extend_from_slice(b" ");
append_fmt(&mut output, format_args!("{:>5.3}", ratio));
output.extend_from_slice(b" ");
} else {
output.extend_from_slice(b" ");
}
append_padded_bytes(&mut output, check, 5);
output.extend_from_slice(b" ");
append_fmt(&mut output, format_args!("{} files\n", total.nbFiles));
output
}
/// Run the policy and ordering around one C-owned `--list` file operation.
///
/// C retains the stat/open and frame-analysis callback, all user-visible
/// status diagnostics, and the full metadata row formatter. Rust preserves
/// C retains the stat/open and frame-analysis callback and all user-visible
/// status diagnostics. Rust owns the metadata row formatter. Rust preserves
/// the original status gates: fatal statuses are reported and return `1`, a
/// frame error reports a diagnostic but still formats and publishes its
/// partial metadata, and success formats and publishes normally.
@@ -972,9 +1124,9 @@ pub unsafe extern "C" fn FIO_rust_listFile(
let display_status = callbacks
.displayStatus
.expect("--list status-display callback is required");
let display_info = callbacks
.displayInfo
.expect("--list info-display callback is required");
let write_output = callbacks
.writeOutput
.expect("--list output callback is required");
let mut info = unsafe { std::mem::zeroed::<FIO_fileInfo_t>() };
let status = unsafe { get_info(callbacks.opaque, input_name, &mut info) };
@@ -996,12 +1148,26 @@ pub unsafe extern "C" fn FIO_rust_listFile(
| FIO_RUST_LIST_FILE_ACTION_FILE_ERROR
| FIO_RUST_LIST_FILE_ACTION_TRUNCATED_INPUT => 1,
FIO_RUST_LIST_FILE_ACTION_SUCCESS | FIO_RUST_LIST_FILE_ACTION_FRAME_ERROR => {
unsafe { display_info(callbacks.opaque, input_name, &info, display_level) };
let rendered = format_list_file_info(input_name, &info, display_level);
unsafe {
write_output(
callbacks.opaque,
rendered.as_ptr().cast::<c_char>(),
rendered.len(),
)
};
unsafe { output.write(project_list_file_info(&info)) };
status
}
FIO_RUST_LIST_FILE_ACTION_INVALID => {
unsafe { display_info(callbacks.opaque, input_name, &info, display_level) };
let rendered = format_list_file_info(input_name, &info, display_level);
unsafe {
write_output(
callbacks.opaque,
rendered.as_ptr().cast::<c_char>(),
rendered.len(),
)
};
unsafe { output.write(project_list_file_info(&info)) };
debug_assert!(
action == FIO_RUST_LIST_FILE_ACTION_SUCCESS
@@ -1070,7 +1236,7 @@ pub struct FIO_listMultipleFilesCallbacks_t {
displayNoFilesError: Option<FIO_listMultipleFilesDisplayFn>,
displayHeader: Option<FIO_listMultipleFilesDisplayFn>,
listFile: Option<FIO_listMultipleFilesListFileFn>,
displayTotal: Option<FIO_listMultipleFilesDisplayTotalFn>,
writeOutput: Option<FIO_listFileWriteOutputFn>,
}
#[cfg(feature = "decompression")]
@@ -2645,8 +2811,9 @@ fn add_list_file_info(
/// Run the policy surrounding C's `--list` file operation.
///
/// Rust owns validation, iteration, error aggregation, and the total-row
/// decision. The callbacks retain C's private file-info operations and all
/// user-visible formatting so the private `fileInfo_t` never crosses the ABI.
/// decision. The callbacks retain C's private file-info operations and exact
/// status diagnostics; Rust owns all row formatting so the private
/// `fileInfo_t` never crosses the ABI.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_listMultipleFiles(
numFiles: c_uint,
@@ -2714,10 +2881,17 @@ pub unsafe extern "C" fn FIO_rust_listMultipleFiles(
}
if numFiles > 1 && displayLevel <= 2 {
let display_total = callbacks
.displayTotal
.expect("--list total callback is required");
unsafe { display_total(callbacks.opaque, &total) };
let write_output = callbacks
.writeOutput
.expect("--list output callback is required");
let rendered = format_list_total(&total);
unsafe {
write_output(
callbacks.opaque,
rendered.as_ptr().cast::<c_char>(),
rendered.len(),
)
};
}
error
@@ -3863,7 +4037,7 @@ mod tests {
infos: Vec<FIO_listFileInfoProjection_t>,
list_calls: usize,
display_levels: Vec<c_int>,
total: Option<FIO_listFileInfoProjection_t>,
output: Vec<u8>,
}
unsafe extern "C" fn list_multiple_is_stdin(
@@ -3906,13 +4080,16 @@ mod tests {
state.statuses[index]
}
unsafe extern "C" fn list_multiple_display_total(
unsafe extern "C" fn list_multiple_write_output(
opaque: *mut c_void,
total: *const FIO_listFileInfoProjection_t,
data: *const c_char,
size: usize,
) {
let state = unsafe { &mut *opaque.cast::<ListMultipleFilesTestState>() };
state.events.push("total");
state.total = Some(unsafe { *total });
state.events.push("write-output");
state
.output
.extend_from_slice(unsafe { std::slice::from_raw_parts(data.cast::<u8>(), size) });
}
fn list_multiple_callbacks(
@@ -3925,7 +4102,7 @@ mod tests {
displayNoFilesError: Some(list_multiple_display_no_files_error),
displayHeader: Some(list_multiple_display_header),
listFile: Some(list_multiple_list_file),
displayTotal: Some(list_multiple_display_total),
writeOutput: Some(list_multiple_write_output),
}
}
@@ -3935,7 +4112,7 @@ mod tests {
info: FIO_fileInfo_t,
events: Vec<&'static str>,
status_actions: Vec<(c_int, c_int)>,
displayed: Option<(FIO_fileInfo_t, c_int)>,
output: Vec<u8>,
}
unsafe extern "C" fn list_file_get_info(
@@ -3960,15 +4137,16 @@ mod tests {
state.status_actions.push((action, display_level));
}
unsafe extern "C" fn list_file_display_info(
unsafe extern "C" fn list_file_write_output(
opaque: *mut c_void,
_file_name: *const c_char,
info: *const FIO_fileInfo_t,
display_level: c_int,
data: *const c_char,
size: usize,
) {
let state = unsafe { &mut *opaque.cast::<ListFileTestState>() };
state.events.push("display-info");
state.displayed = Some((unsafe { *info }, display_level));
state.events.push("write-output");
state
.output
.extend_from_slice(unsafe { std::slice::from_raw_parts(data.cast::<u8>(), size) });
}
fn list_file_callbacks(state: &mut ListFileTestState) -> FIO_listFileCallbacks_t {
@@ -3976,7 +4154,7 @@ mod tests {
opaque: (state as *mut ListFileTestState).cast(),
getInfo: Some(list_file_get_info),
displayStatus: Some(list_file_display_status),
displayInfo: Some(list_file_display_info),
writeOutput: Some(list_file_write_output),
}
}
@@ -4698,7 +4876,7 @@ mod tests {
assert_eq!(status, 1);
assert_eq!(state.events, vec!["stdin", "stdin", "stdin-error"]);
assert_eq!(state.list_calls, 0);
assert!(state.total.is_none());
assert!(state.output.is_empty());
}
#[test]
@@ -4711,7 +4889,7 @@ mod tests {
assert_eq!(status, 1);
assert_eq!(state.events, vec!["no-files"]);
assert_eq!(state.list_calls, 0);
assert!(state.total.is_none());
assert!(state.output.is_empty());
}
#[test]
@@ -4763,20 +4941,21 @@ mod tests {
assert_eq!(state.display_levels, vec![2, 2, 2]);
assert_eq!(
state.events,
vec!["stdin", "stdin", "stdin", "header", "list", "list", "list", "total"]
);
assert_eq!(
state.total,
Some(FIO_listFileInfoProjection_t {
decompressedSize: 30,
compressedSize: 10,
numActualFrames: 4,
numSkippableFrames: 2,
decompUnavailable: 0,
usesCheck: 1,
nbFiles: 2,
})
vec![
"stdin",
"stdin",
"stdin",
"header",
"list",
"list",
"list",
"write-output"
]
);
assert!(state
.output
.starts_with(b"----------------------------------------------------------------- \n"));
assert!(state.output.ends_with(b"2 files\n"));
}
#[test]
@@ -4804,7 +4983,7 @@ mod tests {
assert_eq!(status, 0);
assert_eq!(state.events, vec!["stdin", "stdin", "list", "list"]);
assert!(state.total.is_none());
assert!(state.output.is_empty());
let single_name = [CString::new("single.zst").unwrap()];
let single_table = [single_name[0].as_ptr()];
@@ -4819,7 +4998,83 @@ mod tests {
assert_eq!(status, 0);
assert_eq!(state.events, vec!["stdin", "header", "list"]);
assert!(state.total.is_none());
assert!(state.output.is_empty());
}
#[test]
fn list_formatters_match_original_column_and_verbose_output() {
let file_name = CString::new("input.zst").unwrap();
let info = FIO_fileInfo_t {
decompressedSize: 100,
compressedSize: 40,
windowSize: 32,
numActualFrames: 2,
numSkippableFrames: 1,
decompUnavailable: 0,
usesCheck: 1,
checksum: [1, 2, 3, 4],
nbFiles: 1,
dictID: 7,
};
let normal_expected = [
b" 3 ".as_slice(),
b" 1 ".as_slice(),
b" 40 B".as_slice(),
b" ".as_slice(),
b" 100 B".as_slice(),
b" 2.500 XXH64 input.zst\n".as_slice(),
]
.concat();
assert_eq!(
format_list_file_info(file_name.as_ptr(), &info, 2),
normal_expected
);
let unavailable = FIO_fileInfo_t {
numActualFrames: 0,
decompUnavailable: 1,
usesCheck: 0,
..info
};
let mut unavailable_expected = b" 1 ".to_vec();
unavailable_expected.extend_from_slice(b" 1 ");
unavailable_expected.extend_from_slice(b" 40 B");
unavailable_expected.extend(std::iter::repeat_n(b' ', 23));
unavailable_expected.extend_from_slice(b" None input.zst\n");
assert_eq!(
format_list_file_info(file_name.as_ptr(), &unavailable, 2),
unavailable_expected
);
let verbose_info = FIO_fileInfo_t {
numActualFrames: 1,
..info
};
assert_eq!(
format_list_file_info(file_name.as_ptr(), &verbose_info, 3),
b"input.zst \n# Zstandard Frames: 1\n# Skippable Frames: 1\nDictID: 7\nWindow Size: 32 B (32 B)\nCompressed Size: 40 B (40 B)\nDecompressed Size: 100 B (100 B)\nRatio: 2.5000\nCheck: XXH64 04030201\n\n"
);
let total = FIO_listFileInfoProjection_t {
decompressedSize: 30,
compressedSize: 10,
numActualFrames: 4,
numSkippableFrames: 2,
decompUnavailable: 0,
usesCheck: 1,
nbFiles: 2,
};
let total_expected = [
b"----------------------------------------------------------------- \n".as_slice(),
b" 6 ".as_slice(),
b" 2 ".as_slice(),
b" 10 B".as_slice(),
b" ".as_slice(),
b" 30 B".as_slice(),
b" 3.000 XXH64 2 files\n".as_slice(),
]
.concat();
assert_eq!(format_list_total(&total), total_expected);
}
#[test]
@@ -4861,14 +5116,11 @@ mod tests {
size_of::<Option<FIO_listMultipleFilesListFileFn>>(),
callback
);
assert_eq!(
size_of::<Option<FIO_listMultipleFilesDisplayTotalFn>>(),
callback
);
assert_eq!(size_of::<Option<FIO_listFileWriteOutputFn>>(), callback);
assert_eq!(offset_of!(FIO_listMultipleFilesCallbacks_t, opaque), 0);
assert_eq!(offset_of!(FIO_listMultipleFilesCallbacks_t, isStdin), word);
assert_eq!(
offset_of!(FIO_listMultipleFilesCallbacks_t, displayTotal),
offset_of!(FIO_listMultipleFilesCallbacks_t, writeOutput),
word + 5 * callback
);
assert_eq!(
@@ -4953,9 +5205,9 @@ mod tests {
state.events,
if publishes_info {
if expected_action == FIO_RUST_LIST_FILE_ACTION_FRAME_ERROR {
vec!["get-info", "display-status", "display-info"]
vec!["get-info", "display-status", "write-output"]
} else {
vec!["get-info", "display-info"]
vec!["get-info", "write-output"]
}
} else {
vec!["get-info", "display-status"]
@@ -4972,9 +5224,9 @@ mod tests {
"status {status} diagnostic action"
);
assert_eq!(
state.displayed,
publishes_info.then_some((info, display_level)),
"status {status} display gate"
!state.output.is_empty(),
publishes_info,
"status {status} output gate"
);
assert_eq!(
output,
@@ -5000,7 +5252,7 @@ mod tests {
word + callback
);
assert_eq!(
offset_of!(FIO_listFileCallbacks_t, displayInfo),
offset_of!(FIO_listFileCallbacks_t, writeOutput),
word + 2 * callback
);
assert_eq!(size_of::<FIO_listFileCallbacks_t>(), word + 3 * callback);