ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move benchmark bridge into Rust

Browse Source 
Move benchmark parameter normalization and dispatch into the Rust CLI archive while retaining the stable C launcher entry point. Keep compact CLI variants benchmark-free and add an explicit helpers feature for the C test generators.

Test Plan:

- cargo test --manifest-path rust/cli/Cargo.toml --no-default-features --features benchmark

- cargo test --manifest-path rust/cli/Cargo.toml --no-default-features --features helpers

- cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets --no-default-features --features cli,compression,decompression,benchmark

- cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets --no-default-features --features helpers

- make -B -C programs -j2 zstd zstd-small zstd-frugal

- make -B -C tests -j2 test-cli-tests

- programs/zstd -b1i1 -q
This commit is contained in:
ddidderr
2026-07-18 03:18:58 +02:00
parent 3a10d86c76
commit 96c7f3b900
6 changed files with 185 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+12 -2
View File
@@ -124,7 +124,7 @@ RUST_CLI_STATICLIB := $(RUST_CLI_TARGET_DIR)/release/libzstd_cli_rs.a
RUST_CLI_STATICLIB_32 := $(RUST_CLI_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_cli_rs.a
RUST_CLI_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --release \
--target-dir $(RUST_CLI_TARGET_DIR) \
--no-default-features --features cli,compression,decompression
--no-default-features --features cli,compression,decompression,benchmark
$(RUST_CLI_STATICLIB): $(RUST_CLI_SOURCES)
$(CARGO) build $(RUST_CLI_CARGO_FLAGS)
@@ -132,6 +132,16 @@ $(RUST_CLI_STATICLIB): $(RUST_CLI_SOURCES)
$(RUST_CLI_STATICLIB_32): $(RUST_CLI_SOURCES)
$(CARGO) build $(RUST_CLI_CARGO_FLAGS) --target $(RUST_TARGET_32)
RUST_NOBENCH_CLI_BUILD_CONFIG := cli-c1-d1-nobench-$(RUST_HUF_MODE)
RUST_NOBENCH_CLI_TARGET_DIR := $(RUST_DIR)/target/$(RUST_NOBENCH_CLI_BUILD_CONFIG)
RUST_NOBENCH_CLI_STATICLIB := $(RUST_NOBENCH_CLI_TARGET_DIR)/release/libzstd_cli_rs.a
RUST_NOBENCH_CLI_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --release \
--target-dir $(RUST_NOBENCH_CLI_TARGET_DIR) \
--no-default-features --features cli,compression,decompression
$(RUST_NOBENCH_CLI_STATICLIB): $(RUST_CLI_SOURCES)
$(CARGO) build $(RUST_NOBENCH_CLI_CARGO_FLAGS)
RUST_DECOMPRESS_BUILD_CONFIG := lib-c0-d1-b0-$(RUST_HUF_MODE)-$(RUST_DEBUG_MODE)
RUST_DECOMPRESS_TARGET_DIR := $(RUST_DIR)/target/$(RUST_DECOMPRESS_BUILD_CONFIG)
RUST_DECOMPRESS_STATICLIB := $(RUST_DECOMPRESS_TARGET_DIR)/release/libzstd_rs.a
@@ -455,7 +465,7 @@ CLEAN += zstd-small zstd-frugal
# requested HUF decoder mode from the C sources while the Rust archive sees it.
ZSTD_SMALL_HUF_CFLAGS := $(filter -DHUF_FORCE_DECOMPRESS_X1% -DHUF_FORCE_DECOMPRESS_X2%,$(CFLAGS))
zstd-small: CFLAGS = -Os -Wl,-s $(ZSTD_SMALL_HUF_CFLAGS)
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c $(RUST_STATICLIB) $(RUST_CLI_STATICLIB)
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c $(RUST_STATICLIB) $(RUST_NOBENCH_CLI_STATICLIB)
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
CLEAN += zstd-decompress
programs/zstdcli.c
-56
View File
@@ -10,73 +10,17 @@
/* The CLI parser and control flow live in rust/src/zstd_cli.rs. Keep this
* translation unit as the stable C entry point used by program launchers. */
#include <stddef.h> /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "../lib/zstd.h"
#ifndef ZSTD_NOBENCH
# include "benchzstd.h" /* BMK_benchFilesAdvanced, BMK_syntheticTest */
#endif
int ZSTD_rust_cli_main(int argCount, const char* const argv[]);
const char* ZSTD_rust_cli_expected_version(void);
int ZSTD_rust_cli_bench(const char* const* fileNames, unsigned nbFiles,
const char* dictFileName,
int startCLevel, int endCLevel,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, unsigned nbSeconds,
size_t blockSize, int nbWorkers, int mode);
const char* ZSTD_rust_cli_expected_version(void)
{
return ZSTD_VERSION_STRING;
}
/* Benchmark bridge for the Rust CLI. Whether benchmarking exists is a C
* preprocessor property (ZSTD_NOBENCH), so the decision stays in this shim:
* the Rust frontend calls in unconditionally, and stripped program variants
* never reference benchmark symbols.
* @return the benchmark result code (>= 0), or -1 when unavailable. */
int ZSTD_rust_cli_bench(const char* const* fileNames, unsigned nbFiles,
const char* dictFileName,
int startCLevel, int endCLevel,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, unsigned nbSeconds,
size_t blockSize, int nbWorkers, int mode)
{
#ifndef ZSTD_NOBENCH
BMK_advancedParams_t advancedParams = BMK_initAdvancedParams();
int startLevel = startCLevel;
int endLevel = endCLevel;
advancedParams.nbSeconds = nbSeconds;
advancedParams.blockSize = blockSize;
advancedParams.nbWorkers = nbWorkers;
advancedParams.mode = (BMK_mode_t)mode;
if (advancedParams.mode == BMK_decodeOnly) {
startLevel = 0;
endLevel = 0;
}
if (startLevel > ZSTD_maxCLevel()) startLevel = ZSTD_maxCLevel();
if (endLevel > ZSTD_maxCLevel()) endLevel = ZSTD_maxCLevel();
if (endLevel < startLevel) endLevel = startLevel;
if (nbFiles == 0) {
/* No input file: benchmark a synthetic sample (lorem generator). */
return BMK_syntheticTest(-1.0, startLevel, endLevel,
compressionParams, displayLevel,
&advancedParams);
}
return BMK_benchFilesAdvanced(fileNames, nbFiles, dictFileName,
startLevel, endLevel,
compressionParams, displayLevel,
&advancedParams);
#else
(void)fileNames; (void)nbFiles; (void)dictFileName;
(void)startCLevel; (void)endCLevel; (void)compressionParams;
(void)displayLevel; (void)nbSeconds; (void)blockSize; (void)nbWorkers;
(void)mode;
return -1;
#endif
}
int main(int argCount, const char* argv[])
{
return ZSTD_rust_cli_main(argCount, argv);
rust/cli/Cargo.toml
+6 -1
View File
@@ -7,13 +7,18 @@ edition = "2021"
crate-type = ["staticlib"]
[features]
default = ["cli", "compression", "decompression"]
default = ["cli", "compression", "decompression", "benchmark"]
# The command-line parser and dispatch layer, which requires the C fileio
# backend at link time. Program archives enable it; C test binaries link a
# helpers-only archive (timefn) built without it.
cli = []
compression = []
decompression = []
helpers = []
# The integrated benchmark archive is optional so zstd-small and zstd-frugal
# can omit every benchmark implementation and its timing/data-generator
# dependencies.
benchmark = ["compression"]
[dependencies]
libc = "0.2"
rust/cli/src/lib.rs
+4 -1
View File
@@ -1,8 +1,10 @@
#[cfg(any(feature = "benchmark", feature = "helpers"))]
#[path = "../../src/benchfn.rs"]
mod benchfn;
#[cfg(feature = "cli")]
#[cfg(all(feature = "cli", feature = "benchmark"))]
#[path = "../../src/benchzstd.rs"]
mod benchzstd;
#[cfg(any(feature = "benchmark", feature = "helpers"))]
#[path = "../../src/datagen.rs"]
mod datagen;
#[cfg(feature = "cli")]
@@ -11,6 +13,7 @@ mod dibio;
#[cfg(feature = "cli")]
#[path = "../../src/fileio_prefs.rs"]
mod fileio_prefs;
#[cfg(any(feature = "benchmark", feature = "helpers"))]
#[path = "../../src/lorem.rs"]
mod lorem;
#[path = "../../src/timefn.rs"]
rust/src/zstd_cli.rs
+162 -25
View File
@@ -10,11 +10,11 @@
//! writes, dictionary loading, streaming, and metadata preservation remain in
//! `programs/fileio.c` for this first migration step.
//!
//! Benchmark mode (`-b`) parses here and dispatches through the
//! `ZSTD_rust_cli_bench` bridge in `programs/zstdcli.c`: the run/timing loop
//! (benchfn, timefn) is Rust, while orchestration and result formatting
//! (`benchzstd.c`) remain C behind the preprocessor-gated bridge, so builds
//! with `ZSTD_NOBENCH` never reference benchmark symbols.
//! Benchmark mode (`-b`) parses and dispatches through the Rust CLI archive.
//! The bridge preserves the C CLI's argument normalization and uses the
//! existing Rust `benchzstd` APIs. Builds without the `benchmark` feature
//! return the same unavailable-build result as `ZSTD_NOBENCH` builds without
//! referencing benchmark symbols.
//!
//! Recursive expansion (`-r`), `--filelist`, the output-directory modes, and
//! `--list` reuse the C `FileNamesTable` helpers from `programs/util.c` and
@@ -104,7 +104,7 @@ struct FIO_ctx_t {
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZSTD_compressionParameters {
pub(crate) struct ZSTD_compressionParameters {
windowLog: u32,
chainLog: u32,
hashLog: u32,
@@ -295,22 +295,6 @@ unsafe extern "C" {
display_level: c_int,
) -> c_int;
/// Benchmark bridge implemented by the `programs/zstdcli.c` shim, which
/// owns the `ZSTD_NOBENCH` preprocessor decision. Returns the benchmark
/// result (>= 0), or -1 when benchmarking is compiled out.
fn ZSTD_rust_cli_bench(
file_names: *const *const c_char,
nb_files: c_uint,
dict_file_name: *const c_char,
start_level: c_int,
end_level: c_int,
compression_params: *const ZSTD_compressionParameters,
display_level: c_int,
nb_seconds: c_uint,
block_size: usize,
nb_workers: c_int,
mode: c_int,
) -> c_int;
/// Narrow bridge to `programs/dibio.c`. The file loader and dictionary
/// algorithms remain on the C/Rust library side of this boundary.
#[cfg(feature = "compression")]
@@ -2374,9 +2358,122 @@ fn run_list(cli: &Cli) -> i32 {
unsafe { FIO_listMultipleFiles(inputs.len() as c_uint, inputs.as_ptr(), cli.display_level) }
}
/// Runs benchmark mode through the C bridge. Level clamping against
/// `ZSTD_maxCLevel()` happens on the C side, where the symbol is always
/// available when benchmarking is compiled in. No input file means a
#[cfg(feature = "benchmark")]
fn normalize_benchmark_levels(
start_c_level: c_int,
end_c_level: c_int,
mode: c_int,
max_c_level: c_int,
) -> (c_int, c_int) {
let mut start_level = start_c_level;
let mut end_level = end_c_level;
if mode == crate::benchzstd::BMK_decodeOnly {
start_level = 0;
end_level = 0;
}
if start_level > max_c_level {
start_level = max_c_level;
}
if end_level > max_c_level {
end_level = max_c_level;
}
if end_level < start_level {
end_level = start_level;
}
(start_level, end_level)
}
/// Benchmark bridge formerly implemented by `programs/zstdcli.c`.
///
/// Keep the exported name because it is a useful program-side ABI boundary,
/// but make the Rust CLI archive own both the normalization and dispatch.
#[cfg(feature = "benchmark")]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cli_bench(
file_names: *const *const c_char,
nb_files: c_uint,
dict_file_name: *const c_char,
start_c_level: c_int,
end_c_level: c_int,
compression_params: *const ZSTD_compressionParameters,
display_level: c_int,
nb_seconds: c_uint,
block_size: usize,
nb_workers: c_int,
mode: c_int,
) -> c_int {
let mut advanced_params = crate::benchzstd::BMK_initAdvancedParams();
advanced_params.nbSeconds = nb_seconds;
advanced_params.blockSize = block_size;
advanced_params.nbWorkers = nb_workers;
advanced_params.mode = mode;
let (start_level, end_level) =
normalize_benchmark_levels(start_c_level, end_c_level, mode, unsafe {
ZSTD_maxCLevel()
});
let compression_params = compression_params.cast();
if nb_files == 0 {
/* No input file: benchmark a synthetic sample (lorem generator). */
unsafe {
crate::benchzstd::BMK_syntheticTest(
-1.0,
start_level,
end_level,
compression_params,
display_level,
&advanced_params,
)
}
} else {
unsafe {
crate::benchzstd::BMK_benchFilesAdvanced(
file_names,
nb_files,
dict_file_name,
start_level,
end_level,
compression_params,
display_level,
&advanced_params,
)
}
}
}
/// `ZSTD_NOBENCH` compatibility bridge for compact CLI archives.
#[cfg(not(feature = "benchmark"))]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cli_bench(
file_names: *const *const c_char,
nb_files: c_uint,
dict_file_name: *const c_char,
start_c_level: c_int,
end_c_level: c_int,
compression_params: *const ZSTD_compressionParameters,
display_level: c_int,
nb_seconds: c_uint,
block_size: usize,
nb_workers: c_int,
mode: c_int,
) -> c_int {
let _ = (
file_names,
nb_files,
dict_file_name,
start_c_level,
end_c_level,
compression_params,
display_level,
nb_seconds,
block_size,
nb_workers,
mode,
);
-1
}
/// Runs benchmark mode through the Rust bridge. No input file means a
/// synthetic-sample benchmark, matching the C CLI.
fn run_bench(cli: &Cli) -> Result<i32, String> {
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
@@ -3359,4 +3456,44 @@ mod tests {
assert_eq!(cli.operation, Operation::Bench);
assert!(cli.bench_decode);
}
#[cfg(feature = "benchmark")]
#[test]
fn benchmark_level_normalization_matches_c_bridge() {
assert_eq!(
normalize_benchmark_levels(100, 4, crate::benchzstd::BMK_both, 22),
(22, 22)
);
assert_eq!(
normalize_benchmark_levels(7, 3, crate::benchzstd::BMK_both, 22),
(7, 7)
);
assert_eq!(
normalize_benchmark_levels(7, 3, crate::benchzstd::BMK_decodeOnly, 22),
(0, 0)
);
}
#[cfg(not(feature = "benchmark"))]
#[test]
fn benchmark_bridge_returns_unavailable_without_feature() {
assert_eq!(
unsafe {
ZSTD_rust_cli_bench(
ptr::null(),
0,
ptr::null(),
3,
3,
ptr::null(),
0,
3,
0,
1,
0,
)
},
-1
);
}
}
tests/Makefile
+1 -1
View File
@@ -117,7 +117,7 @@ RUST_CLI_HELPERS_STATICLIB := $(RUST_CLI_HELPERS_TARGET_DIR)/release/libzstd_cli
RUST_CLI_HELPERS_STATICLIB_32 := $(RUST_CLI_HELPERS_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_cli_rs.a
RUST_CLI_HELPERS_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --release \
--target-dir $(RUST_CLI_HELPERS_TARGET_DIR) \
--no-default-features
--no-default-features --features helpers
$(RUST_CLI_HELPERS_STATICLIB): $(RUST_CLI_HELPER_SOURCES)
$(CARGO) build $(RUST_CLI_HELPERS_CARGO_FLAGS)