feat(rust): port benchmark loop and CLI bench mode
Move the implementation of programs/benchfn.c into rust/src/benchfn.rs and wire benchmark mode (-b/-e/-i) into the Rust CLI frontend, which previously rejected those options as not yet implemented. `zstd -b1 -i0 FILE` and range runs like `zstd -b5e6 -i0 FILE` work again, including the synthetic-sample benchmark when no file is given. benchfn.rs is a faithful port of the run/timing state machine: BMK_benchFunction keeps the exact loop accounting (first-loop blockResults and errorFn checks, dstSize summed on the first loop only, 0xE5 warm-up of result buffers, nbLoops minimum of 1) and BMK_benchTimedFn keeps the same convergence behavior (x10 workload growth for short runs, budget-based nbLoops estimation, runs below half the run budget re-tried rather than reported, best qualifying run returned). Arithmetic that C leaves to unsigned wrap-around uses wrapping operations so debug builds cannot panic where release C would wrap. ABI notes: BMK_runTime_t and BMK_runOutcome_t are returned by value across the C boundary and BMK_benchParams_t is passed by value, so all three are repr(C) mirrors of benchfn.h; their field offsets are pinned by const asserts in Rust and matching C static asserts in the benchfn.c shim, which is now declaration-only. BMK_timedFnState_t stays opaque, fits the 64-byte BMK_timedFnState_shell (compile-time checked), and is malloc/free-managed so creation and destruction remain interchangeable with C callers. The CLI parses -b (bench mode), -e (range end, digits attach directly, defaulting to 0 like readU32FromChar) and -i (duration in seconds), then dispatches through a new ZSTD_rust_cli_bench bridge in the zstdcli.c shim. The bridge exists because benchmark availability is a C preprocessor property (ZSTD_NOBENCH): orchestration and reporting stay in C benchzstd.c, stripped variants (zstd-small, zstd-compress, zstd-decompress) compile the stub branch and report "benchmark mode is not available in this build", and the Rust side never references benchmark symbols directly. Level clamping against ZSTD_maxCLevel() happens in the bridge, where the symbol is guaranteed to exist whenever benchmarking is compiled in. -T selects the worker count, defaulting to single-threaded like the C bench path; -S (separate files) and --priority=rt remain unimplemented. Makefile updates only extend the Rust source prerequisite lists with benchfn.rs; the helpers-archive plumbing from the timefn commit already links fullbench(-lib/-dll/32) and paramgrill, the benchfn consumers among the C tests. Original C test sources are untouched. Known pre-existing issues, unchanged by this commit: tests/fullbench-lib fails to link at the base commit too (libzstd.a precedes fullbench.c in its link line), and the cli-tests basic/help.sh, compression/levels.sh, compression/golden.sh, and decompression/pass-through.sh scripts fail identically with a base-commit binary because the Rust CLI frontend is still a partial reimplementation. Test Plan: - cd rust && cargo fmt --check && cargo clippy --all-targets -- -D warnings && cargo test --all-targets && cargo build --release - cd rust/cli && cargo fmt --check && cargo clippy --all-targets -- -D warnings && cargo test --all-targets; repeat tests with --no-default-features plus features compression / decompression / (none) - make -C programs zstd; ./programs/zstd -b1 -i0 lib/common/xxhash.c; ./programs/zstd -b5e6 -i0 programs/fileio.c; ./programs/zstd -b1 -i0 (synthetic); echo roundtrip via zstd | zstd -d - make -C programs zstd-small zstd-compress zstd-decompress zstd-nolegacy zstd-dictBuilder; zstd-small -b reports benchmark unavailable; compress/ decompress roundtrip across the split binaries - make -C tests fullbench fuzzer zstreamtest paramgrill decodecorpus poolTests fullbench32 fuzzer32; ./tests/fullbench -i0 (exercises Rust BMK_benchTimedFn from C); ./tests/fullbench32 -i0; ./tests/fuzzer -i1 --no-big-tests; ./tests/poolTests; make -C tests test-rust-lib-smoke - cli-tests subset: basic/version.sh, compression/basic.sh, compression/multiple-files.sh pass; failing scripts match the base commit Refs: rust/README.md
This commit is contained in:
+147
-5
@@ -10,9 +10,15 @@
|
||||
//! writes, dictionary loading, streaming, and metadata preservation remain in
|
||||
//! `programs/fileio.c` for this first migration step.
|
||||
//!
|
||||
//! Benchmark mode (`-b`) parses here and dispatches through the
|
||||
//! `ZSTD_rust_cli_bench` bridge in `programs/zstdcli.c`: the run/timing loop
|
||||
//! (benchfn, timefn) is Rust, while orchestration and result formatting
|
||||
//! (`benchzstd.c`) remain C behind the preprocessor-gated bridge, so builds
|
||||
//! with `ZSTD_NOBENCH` never reference benchmark symbols.
|
||||
//!
|
||||
//! Remaining C-only CLI boundaries are called out in `unsupported()` below:
|
||||
//! benchmark execution, dictionary training, recursive/file-list expansion,
|
||||
//! tracing, alternate-format selection, and the advanced directory modes.
|
||||
//! dictionary training, recursive/file-list expansion, tracing,
|
||||
//! alternate-format selection, and the advanced directory modes.
|
||||
|
||||
use std::env;
|
||||
use std::ffi::{CStr, CString, OsStr, OsString};
|
||||
@@ -30,6 +36,7 @@ use std::os::unix::fs::FileTypeExt;
|
||||
const DEFAULT_CLEVEL: i32 = 3;
|
||||
#[cfg(feature = "compression")]
|
||||
const DEFAULT_MAX_CLEVEL: i32 = 19;
|
||||
const DEFAULT_BENCH_NB_SECONDS: u32 = 3;
|
||||
const DEFAULT_MEM_LIMIT: u32 = 1 << 27;
|
||||
const DEFAULT_LONG_WINDOW_LOG: u32 = 27;
|
||||
const MAX_FAST_ACCELERATION: i32 = 128 << 10;
|
||||
@@ -173,6 +180,22 @@ unsafe extern "C" {
|
||||
output: *const c_char,
|
||||
dict: *const c_char,
|
||||
) -> c_int;
|
||||
|
||||
/// Benchmark bridge implemented by the `programs/zstdcli.c` shim, which
|
||||
/// owns the `ZSTD_NOBENCH` preprocessor decision. Returns the benchmark
|
||||
/// result (>= 0), or -1 when benchmarking is compiled out.
|
||||
fn ZSTD_rust_cli_bench(
|
||||
file_names: *const *const c_char,
|
||||
nb_files: c_uint,
|
||||
dict_file_name: *const c_char,
|
||||
start_level: c_int,
|
||||
end_level: c_int,
|
||||
compression_params: *const ZSTD_compressionParameters,
|
||||
display_level: c_int,
|
||||
nb_seconds: c_uint,
|
||||
block_size: usize,
|
||||
nb_workers: c_int,
|
||||
) -> c_int;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -180,6 +203,7 @@ enum Operation {
|
||||
Compress,
|
||||
Decompress,
|
||||
Test,
|
||||
Bench,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -230,6 +254,8 @@ struct Cli {
|
||||
row_match_finder: i32,
|
||||
exclude_compressed: bool,
|
||||
compression_params: ZSTD_compressionParameters,
|
||||
bench_end_level: Option<i32>,
|
||||
bench_nb_seconds: Option<u32>,
|
||||
unsupported_program: Option<String>,
|
||||
}
|
||||
|
||||
@@ -275,6 +301,8 @@ impl Cli {
|
||||
row_match_finder: ZSTD_PS_AUTO,
|
||||
exclude_compressed: false,
|
||||
compression_params: ZSTD_compressionParameters::default(),
|
||||
bench_end_level: None,
|
||||
bench_nb_seconds: None,
|
||||
unsupported_program: None,
|
||||
};
|
||||
|
||||
@@ -404,9 +432,22 @@ fn usage(advanced: bool) {
|
||||
out,
|
||||
" --adapt[=min=#,max=#], --rsyncable, --[no-]row-match-finder"
|
||||
);
|
||||
let _ = writeln!(out, "\nBenchmark options:");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"\nNot yet migrated: benchmark, dictionary training, recursive/file-list expansion,"
|
||||
" -b# Benchmark file(s) at compression level #"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" -e# Test all levels from -b# up to # included"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" -i# Set the minimum evaluation time to # seconds"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"\nNot yet migrated: dictionary training, recursive/file-list expansion,"
|
||||
);
|
||||
let _ = writeln!(out, "trace, alternate formats, and output-directory modes.");
|
||||
}
|
||||
@@ -905,6 +946,33 @@ fn parse_short_options(
|
||||
'd' => cli.operation = Operation::Decompress,
|
||||
'z' => cli.operation = Operation::Compress,
|
||||
't' => cli.operation = Operation::Test,
|
||||
'b' => cli.operation = Operation::Bench,
|
||||
'e' | 'i' => {
|
||||
// Benchmark range end (-e#) and duration (-i#): like the C
|
||||
// parser, digits attach directly and default to 0.
|
||||
let mut digits_end = offset + 1;
|
||||
while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
|
||||
digits_end += 1;
|
||||
}
|
||||
let digits = &value[offset + 1..digits_end];
|
||||
if option == 'e' {
|
||||
cli.bench_end_level = Some(if digits.is_empty() {
|
||||
0
|
||||
} else {
|
||||
parse_i32(digits, "benchmark end level")?
|
||||
});
|
||||
} else {
|
||||
cli.bench_nb_seconds = Some(if digits.is_empty() {
|
||||
0
|
||||
} else {
|
||||
digits
|
||||
.parse::<u32>()
|
||||
.map_err(|_| format!("invalid benchmark duration: {digits:?}"))?
|
||||
});
|
||||
}
|
||||
offset = digits_end;
|
||||
continue;
|
||||
}
|
||||
'c' => {
|
||||
cli.output = Some(cstring(STDOUT_MARK)?);
|
||||
cli.force_stdout = true;
|
||||
@@ -940,7 +1008,7 @@ fn parse_short_options(
|
||||
}
|
||||
break;
|
||||
}
|
||||
'b' | 'e' | 'i' | 'l' | 'p' | 'P' | 'r' | 's' | 'S' => {
|
||||
'l' | 'p' | 'P' | 'r' | 's' | 'S' => {
|
||||
unsupported(&format!("-{option}"))?;
|
||||
}
|
||||
_ => return Err(format!("unknown option -{option}")),
|
||||
@@ -1231,16 +1299,52 @@ unsafe fn run_decompress(
|
||||
dictionary,
|
||||
)
|
||||
},
|
||||
Operation::Compress => unreachable!("compression is dispatched separately"),
|
||||
Operation::Compress | Operation::Bench => {
|
||||
unreachable!("compression and benchmark are dispatched separately")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs benchmark mode through the C bridge. Level clamping against
|
||||
/// `ZSTD_maxCLevel()` happens on the C side, where the symbol is always
|
||||
/// available when benchmarking is compiled in. No input file means a
|
||||
/// synthetic-sample benchmark, matching the C CLI.
|
||||
fn run_bench(cli: &Cli) -> Result<i32, String> {
|
||||
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
|
||||
let dictionary = cli
|
||||
.dictionary
|
||||
.as_ref()
|
||||
.map_or(ptr::null(), |value| value.as_ptr());
|
||||
let result = unsafe {
|
||||
ZSTD_rust_cli_bench(
|
||||
inputs.as_ptr(),
|
||||
inputs.len() as c_uint,
|
||||
dictionary,
|
||||
cli.level,
|
||||
cli.bench_end_level.unwrap_or(cli.level),
|
||||
&cli.compression_params,
|
||||
cli.display_level,
|
||||
cli.bench_nb_seconds.unwrap_or(DEFAULT_BENCH_NB_SECONDS),
|
||||
cli.block_size.unwrap_or(0),
|
||||
// The C CLI benchmarks single-threaded unless -T was given.
|
||||
cli.workers.unwrap_or(1),
|
||||
)
|
||||
};
|
||||
if result < 0 {
|
||||
return Err("benchmark mode is not available in this build".to_owned());
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn run_cli(mut cli: Cli) -> Result<i32, String> {
|
||||
if let Some(program_name) = &cli.unsupported_program {
|
||||
return Err(format!(
|
||||
"{program_name} compatibility mode is not yet implemented by the Rust CLI frontend"
|
||||
));
|
||||
}
|
||||
if cli.operation == Operation::Bench {
|
||||
return run_bench(&cli);
|
||||
}
|
||||
let explicit_input_count = cli.inputs.len();
|
||||
filter_symlink_inputs(&mut cli);
|
||||
if explicit_input_count > 0 && cli.inputs.is_empty() {
|
||||
@@ -1347,6 +1451,7 @@ fn run_cli(mut cli: Cli) -> Result<i32, String> {
|
||||
#[cfg(not(feature = "decompression"))]
|
||||
unreachable!("unsupported decompression was rejected above")
|
||||
}
|
||||
Operation::Bench => unreachable!("benchmark mode was dispatched earlier"),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1581,4 +1686,41 @@ mod tests {
|
||||
|
||||
assert!(error.contains("not yet implemented"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bench_mode_parses_level_duration_and_defaults() {
|
||||
let cli = parse(&["zstd", "-b1", "-i0", "input"]);
|
||||
|
||||
assert_eq!(cli.operation, Operation::Bench);
|
||||
assert_eq!(cli.level, 1);
|
||||
assert_eq!(cli.bench_nb_seconds, Some(0));
|
||||
assert_eq!(cli.bench_end_level, None);
|
||||
assert_eq!(
|
||||
cli.inputs
|
||||
.iter()
|
||||
.map(|input| input.as_bytes())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![&b"input"[..]]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bench_range_aggregates_within_a_single_argument() {
|
||||
let cli = parse(&["zstd", "-b5e6i2", "input"]);
|
||||
|
||||
assert_eq!(cli.operation, Operation::Bench);
|
||||
assert_eq!(cli.level, 5);
|
||||
assert_eq!(cli.bench_end_level, Some(6));
|
||||
assert_eq!(cli.bench_nb_seconds, Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bench_duration_without_digits_defaults_to_zero() {
|
||||
let cli = parse(&["zstd", "-b", "-e", "-i"]);
|
||||
|
||||
assert_eq!(cli.operation, Operation::Bench);
|
||||
assert_eq!(cli.level, DEFAULT_CLEVEL);
|
||||
assert_eq!(cli.bench_end_level, Some(0));
|
||||
assert_eq!(cli.bench_nb_seconds, Some(0));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user