diff --git a/programs/Makefile b/programs/Makefile index 020ea6837..eb76c1dc1 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -33,6 +33,7 @@ RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \ $(shell find $(RUST_DIR)/src -type f -name '*.rs' -print) RUST_CLI_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \ $(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs \ + $(RUST_DIR)/src/benchzstd.rs \ $(RUST_DIR)/src/fileio_prefs.rs \ $(RUST_DIR)/src/dibio.rs \ $(RUST_DIR)/src/util.rs \ @@ -231,7 +232,7 @@ ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC)) ZSTDLIB_LOCAL_OBJ0 := $(ZSTDLIB_LOCAL_SRC:.c=.o) ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_OBJ0:.S=.o) -ZSTD_CLI_SRC := $(sort $(wildcard *.c)) +ZSTD_CLI_SRC := $(filter-out benchzstd.c,$(sort $(wildcard *.c))) ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o) ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC) diff --git a/rust/cli/src/lib.rs b/rust/cli/src/lib.rs index 8e3fd53e5..8c6fd19cc 100644 --- a/rust/cli/src/lib.rs +++ b/rust/cli/src/lib.rs @@ -1,5 +1,8 @@ #[path = "../../src/benchfn.rs"] mod benchfn; +#[cfg(feature = "cli")] +#[path = "../../src/benchzstd.rs"] +mod benchzstd; #[path = "../../src/datagen.rs"] mod datagen; #[cfg(feature = "cli")] diff --git a/rust/src/benchzstd.rs b/rust/src/benchzstd.rs new file mode 100644 index 000000000..8e2276799 --- /dev/null +++ b/rust/src/benchzstd.rs @@ -0,0 +1,1796 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::too_many_arguments)] + +//! Benchmark orchestration for the command-line programs. +//! +//! This is the Rust counterpart of `programs/benchzstd.c`. The public +//! structures intentionally mirror `programs/benchzstd.h` instead of using +//! Rust-only enums or references: they cross the C ABI by value or through +//! raw pointers. The actual codec calls use the public zstd context API, so +//! this module does not depend on the private `ZSTD_CCtx_s` or `ZSTD_DCtx_s` +//! layouts. +//! +//! The module is kept independent of the CLI dispatcher. The full CLI archive +//! includes it and the program build omits `benchzstd.c`; the existing +//! `benchfn.rs`, `datagen.rs`, and `lorem.rs` symbols are used through their +//! stable C ABI in that configuration. + +use std::ffi::{c_char, c_void, CStr}; +use std::fs; +use std::io::{self, Read, Write}; +use std::mem::{offset_of, size_of}; +use std::os::raw::{c_int, c_uint}; +use std::path::PathBuf; +use std::ptr; + +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; + +pub const MB_UNIT: u64 = 1_000_000; +pub const BMK_TIMETEST_DEFAULT_S: c_uint = 3; +#[cfg(not(test))] +const BMK_RUNTEST_DEFAULT_MS: c_uint = 1_000; +#[cfg(test)] +const ZSTD_MAX_CLEVEL_FALLBACK: c_int = 22; +const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX; +const ZSTD_CONTENTSIZE_ERROR: u64 = u64::MAX - 1; +#[cfg(not(test))] +const ZSTD_RESET_SESSION_AND_PARAMETERS: c_int = 3; +const ZSTD_MAX_ERROR_CODE: c_int = 120; +#[cfg(test)] +const ZSTD_ERROR_GENERIC: c_int = 1; +const ZSTD_ERROR_DST_SIZE_TOO_SMALL: c_int = 70; + +/* Public and experimental parameter values from zstd.h. */ +const ZSTD_C_COMPRESSION_LEVEL: c_int = 100; +const ZSTD_C_WINDOW_LOG: c_int = 101; +const ZSTD_C_HASH_LOG: c_int = 102; +const ZSTD_C_CHAIN_LOG: c_int = 103; +const ZSTD_C_SEARCH_LOG: c_int = 104; +const ZSTD_C_MIN_MATCH: c_int = 105; +const ZSTD_C_TARGET_LENGTH: c_int = 106; +const ZSTD_C_STRATEGY: c_int = 107; +const ZSTD_C_TARGET_C_BLOCK_SIZE: c_int = 130; +const ZSTD_C_NB_WORKERS: c_int = 400; +const ZSTD_C_ENABLE_LDM: c_int = 160; +const ZSTD_C_LDM_HASH_LOG: c_int = 161; +const ZSTD_C_LDM_MIN_MATCH: c_int = 162; +const ZSTD_C_LDM_BUCKET_SIZE_LOG: c_int = 163; +const ZSTD_C_LDM_HASH_RATE_LOG: c_int = 164; +const ZSTD_C_LITERAL_COMPRESSION_MODE: c_int = 1002; +const ZSTD_C_USE_ROW_MATCH_FINDER: c_int = 1011; + +/// `BMK_mode_t` is a C enum and therefore has the ABI of `int` here. +pub type BMK_mode_t = c_int; +pub const BMK_both: BMK_mode_t = 0; +pub const BMK_decodeOnly: BMK_mode_t = 1; +pub const BMK_compressOnly: BMK_mode_t = 2; + +/// `ZSTD_ParamSwitch_e` is represented as an `int` in the C ABI. +pub type ZSTD_ParamSwitch_e = c_int; +pub const ZSTD_ps_auto: ZSTD_ParamSwitch_e = 0; +#[allow(dead_code)] +pub const ZSTD_ps_enable: ZSTD_ParamSwitch_e = 1; +#[allow(dead_code)] +pub const ZSTD_ps_disable: ZSTD_ParamSwitch_e = 2; + +/// ABI-compatible `ZSTD_compressionParameters` from `zstd.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ZSTD_compressionParameters { + pub windowLog: u32, + pub chainLog: u32, + pub hashLog: u32, + pub searchLog: u32, + pub minMatch: u32, + pub targetLength: u32, + pub strategy: c_int, +} + +/// ABI-compatible `BMK_benchResult_t` from `benchzstd.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct BMK_benchResult_t { + pub cSize: usize, + pub cSpeed: u64, + pub dSpeed: u64, + pub cMem: usize, +} + +/// Opaque result variant from `benchzstd.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct BMK_benchOutcome_t { + pub internal_never_ever_use_directly: BMK_benchResult_t, + pub tag: c_int, +} + +/// ABI-compatible advanced benchmark controls. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BMK_advancedParams_t { + pub mode: BMK_mode_t, + pub nbSeconds: c_uint, + pub blockSize: usize, + pub targetCBlockSize: usize, + pub nbWorkers: c_int, + pub realTime: c_uint, + pub additionalParam: c_int, + pub ldmFlag: c_int, + pub ldmMinMatch: c_int, + pub ldmHashLog: c_int, + pub ldmBucketSizeLog: c_int, + pub ldmHashRateLog: c_int, + pub literalCompressionMode: ZSTD_ParamSwitch_e, + pub useRowMatchFinder: c_int, +} + +impl Default for BMK_advancedParams_t { + fn default() -> Self { + Self { + mode: BMK_both, + nbSeconds: BMK_TIMETEST_DEFAULT_S, + blockSize: 0, + targetCBlockSize: 0, + nbWorkers: 0, + realTime: 0, + additionalParam: 0, + ldmFlag: 0, + ldmMinMatch: 0, + ldmHashLog: 0, + ldmBucketSizeLog: 0, + ldmHashRateLog: 0, + literalCompressionMode: ZSTD_ps_auto, + useRowMatchFinder: 0, + } + } +} + +/* Keep the by-value C ABI honest on both 32- and 64-bit targets. */ +const _: () = assert!(offset_of!(BMK_benchResult_t, cSpeed) == size_of::()); +const _: () = + assert!(offset_of!(BMK_benchResult_t, dSpeed) == size_of::() + size_of::()); +const _: () = assert!(offset_of!(BMK_benchOutcome_t, tag) == size_of::()); +const _: () = assert!(size_of::() == 7 * size_of::()); +const _: () = assert!(offset_of!(ZSTD_compressionParameters, strategy) == 6 * size_of::()); +const _: () = assert!(offset_of!(BMK_advancedParams_t, nbSeconds) == size_of::()); +const _: () = assert!(offset_of!(BMK_advancedParams_t, blockSize) == 2 * size_of::()); +const _: () = assert!( + offset_of!(BMK_advancedParams_t, targetCBlockSize) + == 2 * size_of::() + size_of::() +); + +/* The following mirrors are private to this module and are used only to + * call the already migrated benchfn implementation by value. */ +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct BMK_runTime_t { + nanoSecPerRun: f64, + sumOfReturn: usize, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct BMK_runOutcome_t { + internal_never_ever_use_directly: BMK_runTime_t, + error_result_never_ever_use_directly: usize, + error_tag_never_ever_use_directly: c_int, +} + +type BMK_benchFn_t = + Option usize>; +type BMK_initFn_t = Option usize>; +type BMK_errorFn_t = Option c_uint>; + +#[repr(C)] +#[derive(Clone, Copy)] +struct BMK_benchParams_t { + benchFn: BMK_benchFn_t, + benchPayload: *mut c_void, + initFn: BMK_initFn_t, + initPayload: *mut c_void, + errorFn: BMK_errorFn_t, + blockCount: usize, + srcBuffers: *const *const c_void, + srcSizes: *const usize, + dstBuffers: *const *mut c_void, + dstCapacities: *const usize, + blockResults: *mut usize, +} + +#[repr(C)] +struct BMK_timedFnState_t { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct ZSTD_inBuffer { + src: *const c_void, + size: usize, + pos: usize, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct ZSTD_outBuffer { + dst: *mut c_void, + size: usize, + pos: usize, +} + +#[cfg(not(test))] +unsafe extern "C" { + fn ZSTD_maxCLevel() -> c_int; + fn ZSTD_compressBound(srcSize: usize) -> usize; + fn ZSTD_findDecompressedSize(src: *const c_void, srcSize: usize) -> u64; + fn ZSTD_createCCtx() -> *mut c_void; + fn ZSTD_freeCCtx(cctx: *mut c_void) -> usize; + fn ZSTD_CCtx_reset(cctx: *mut c_void, reset: c_int) -> usize; + fn ZSTD_CCtx_setParameter(cctx: *mut c_void, param: c_int, value: c_int) -> usize; + fn ZSTD_CCtx_loadDictionary(cctx: *mut c_void, dict: *const c_void, dictSize: usize) -> usize; + fn ZSTD_compress2( + cctx: *mut c_void, + dst: *mut c_void, + dstCapacity: usize, + src: *const c_void, + srcSize: usize, + ) -> usize; + fn ZSTD_sizeof_CCtx(cctx: *const c_void) -> usize; + fn ZSTD_createDCtx() -> *mut c_void; + fn ZSTD_freeDCtx(dctx: *mut c_void) -> usize; + fn ZSTD_DCtx_reset(dctx: *mut c_void, reset: c_int) -> usize; + fn ZSTD_DCtx_loadDictionary(dctx: *mut c_void, dict: *const c_void, dictSize: usize) -> usize; + fn ZSTD_decompressStream( + dctx: *mut c_void, + output: *mut ZSTD_outBuffer, + input: *mut ZSTD_inBuffer, + ) -> usize; + fn ZSTD_versionString() -> *const c_char; + + fn BMK_createTimedFnState(total_ms: c_uint, run_ms: c_uint) -> *mut BMK_timedFnState_t; + fn BMK_freeTimedFnState(state: *mut BMK_timedFnState_t); + fn BMK_benchTimedFn( + state: *mut BMK_timedFnState_t, + params: BMK_benchParams_t, + ) -> BMK_runOutcome_t; + fn BMK_isCompleted_TimedFn(state: *const BMK_timedFnState_t) -> c_int; + + fn RDG_genBuffer( + buffer: *mut c_void, + size: usize, + matchProba: f64, + litProba: f64, + seed: c_uint, + ); + fn LOREM_genBuffer(buffer: *mut c_void, size: usize, seed: c_uint); +} + +#[inline] +fn error_value(code: c_int) -> usize { + (0usize).wrapping_sub(code as usize) +} + +#[inline] +fn is_zstd_error(result: usize) -> bool { + result > error_value(ZSTD_MAX_ERROR_CODE) +} + +#[cfg(not(test))] +unsafe fn zstd_max_c_level() -> c_int { + unsafe { ZSTD_maxCLevel() } +} + +#[cfg(test)] +unsafe fn zstd_max_c_level() -> c_int { + ZSTD_MAX_CLEVEL_FALLBACK +} + +#[cfg(not(test))] +unsafe fn zstd_compress_bound(size: usize) -> usize { + unsafe { ZSTD_compressBound(size) } +} + +#[cfg(test)] +unsafe fn zstd_compress_bound(size: usize) -> usize { + size.saturating_add(64) +} + +#[cfg(not(test))] +unsafe fn zstd_find_decompressed_size(src: *const c_void, size: usize) -> u64 { + unsafe { ZSTD_findDecompressedSize(src, size) } +} + +#[cfg(test)] +unsafe fn zstd_find_decompressed_size(_src: *const c_void, _size: usize) -> u64 { + ZSTD_CONTENTSIZE_ERROR +} + +#[cfg(not(test))] +unsafe fn zstd_create_cctx() -> *mut c_void { + unsafe { ZSTD_createCCtx() } +} + +#[cfg(test)] +unsafe fn zstd_create_cctx() -> *mut c_void { + ptr::dangling_mut::() +} + +#[cfg(not(test))] +unsafe fn zstd_free_cctx(cctx: *mut c_void) { + unsafe { ZSTD_freeCCtx(cctx) }; +} + +#[cfg(test)] +unsafe fn zstd_free_cctx(_cctx: *mut c_void) {} + +#[cfg(not(test))] +unsafe fn zstd_create_dctx() -> *mut c_void { + unsafe { ZSTD_createDCtx() } +} + +#[cfg(test)] +unsafe fn zstd_create_dctx() -> *mut c_void { + ptr::dangling_mut::() +} + +#[cfg(not(test))] +unsafe fn zstd_free_dctx(dctx: *mut c_void) { + unsafe { ZSTD_freeDCtx(dctx) }; +} + +#[cfg(test)] +unsafe fn zstd_free_dctx(_dctx: *mut c_void) {} + +#[cfg(not(test))] +unsafe fn zstd_cctx_reset(cctx: *mut c_void) -> usize { + unsafe { ZSTD_CCtx_reset(cctx, ZSTD_RESET_SESSION_AND_PARAMETERS) } +} + +#[cfg(test)] +unsafe fn zstd_cctx_reset(_cctx: *mut c_void) -> usize { + 0 +} + +#[cfg(not(test))] +unsafe fn zstd_cctx_set_parameter(cctx: *mut c_void, param: c_int, value: c_int) -> usize { + unsafe { ZSTD_CCtx_setParameter(cctx, param, value) } +} + +#[cfg(test)] +unsafe fn zstd_cctx_set_parameter(_cctx: *mut c_void, _param: c_int, _value: c_int) -> usize { + 0 +} + +#[cfg(not(test))] +unsafe fn zstd_cctx_load_dictionary(cctx: *mut c_void, dict: *const c_void, size: usize) -> usize { + unsafe { ZSTD_CCtx_loadDictionary(cctx, dict, size) } +} + +#[cfg(test)] +unsafe fn zstd_cctx_load_dictionary( + _cctx: *mut c_void, + _dict: *const c_void, + _size: usize, +) -> usize { + 0 +} + +#[cfg(not(test))] +unsafe fn zstd_compress2( + cctx: *mut c_void, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, +) -> usize { + unsafe { ZSTD_compress2(cctx, dst, dst_capacity, src, src_size) } +} + +#[cfg(test)] +unsafe fn zstd_compress2( + _cctx: *mut c_void, + _dst: *mut c_void, + _dst_capacity: usize, + _src: *const c_void, + _src_size: usize, +) -> usize { + error_value(ZSTD_ERROR_GENERIC) +} + +#[cfg(not(test))] +unsafe fn zstd_sizeof_cctx(cctx: *const c_void) -> usize { + unsafe { ZSTD_sizeof_CCtx(cctx) } +} + +#[cfg(test)] +unsafe fn zstd_sizeof_cctx(_cctx: *const c_void) -> usize { + 0 +} + +#[cfg(not(test))] +unsafe fn zstd_dctx_reset(dctx: *mut c_void) -> usize { + unsafe { ZSTD_DCtx_reset(dctx, ZSTD_RESET_SESSION_AND_PARAMETERS) } +} + +#[cfg(test)] +unsafe fn zstd_dctx_reset(_dctx: *mut c_void) -> usize { + 0 +} + +#[cfg(not(test))] +unsafe fn zstd_dctx_load_dictionary(dctx: *mut c_void, dict: *const c_void, size: usize) -> usize { + unsafe { ZSTD_DCtx_loadDictionary(dctx, dict, size) } +} + +#[cfg(test)] +unsafe fn zstd_dctx_load_dictionary( + _dctx: *mut c_void, + _dict: *const c_void, + _size: usize, +) -> usize { + 0 +} + +#[cfg(not(test))] +unsafe fn zstd_decompress_stream( + dctx: *mut c_void, + output: *mut ZSTD_outBuffer, + input: *mut ZSTD_inBuffer, +) -> usize { + unsafe { ZSTD_decompressStream(dctx, output, input) } +} + +#[cfg(test)] +unsafe fn zstd_decompress_stream( + _dctx: *mut c_void, + _output: *mut ZSTD_outBuffer, + _input: *mut ZSTD_inBuffer, +) -> usize { + error_value(ZSTD_ERROR_GENERIC) +} + +#[cfg(not(test))] +unsafe fn zstd_version() -> String { + let ptr = unsafe { ZSTD_versionString() }; + if ptr.is_null() { + String::new() + } else { + unsafe { CStr::from_ptr(ptr) } + .to_string_lossy() + .into_owned() + } +} + +#[cfg(test)] +unsafe fn zstd_version() -> String { + "test".to_owned() +} + +#[cfg(not(test))] +unsafe fn timed_state_create(total_ms: c_uint) -> *mut BMK_timedFnState_t { + unsafe { BMK_createTimedFnState(total_ms, BMK_RUNTEST_DEFAULT_MS) } +} + +#[cfg(test)] +unsafe fn timed_state_create(_total_ms: c_uint) -> *mut BMK_timedFnState_t { + ptr::null_mut() +} + +#[cfg(not(test))] +unsafe fn timed_state_free(state: *mut BMK_timedFnState_t) { + unsafe { BMK_freeTimedFnState(state) }; +} + +#[cfg(test)] +unsafe fn timed_state_free(_state: *mut BMK_timedFnState_t) {} + +#[cfg(not(test))] +unsafe fn timed_state_run( + state: *mut BMK_timedFnState_t, + params: BMK_benchParams_t, +) -> BMK_runOutcome_t { + unsafe { BMK_benchTimedFn(state, params) } +} + +#[cfg(test)] +unsafe fn timed_state_run( + _state: *mut BMK_timedFnState_t, + _params: BMK_benchParams_t, +) -> BMK_runOutcome_t { + BMK_runOutcome_t { + internal_never_ever_use_directly: BMK_runTime_t::default(), + error_result_never_ever_use_directly: error_value(ZSTD_ERROR_GENERIC), + error_tag_never_ever_use_directly: 1, + } +} + +#[cfg(not(test))] +unsafe fn timed_state_complete(state: *const BMK_timedFnState_t) -> bool { + unsafe { BMK_isCompleted_TimedFn(state) != 0 } +} + +#[cfg(test)] +unsafe fn timed_state_complete(_state: *const BMK_timedFnState_t) -> bool { + true +} + +#[cfg(not(test))] +unsafe fn generate_rdg(buffer: *mut c_void, size: usize, compressibility: f64) { + unsafe { RDG_genBuffer(buffer, size, compressibility, 0.0, 0) }; +} + +#[cfg(test)] +unsafe fn generate_rdg(buffer: *mut c_void, size: usize, _compressibility: f64) { + if size != 0 { + unsafe { ptr::write_bytes(buffer.cast::(), b'0', size) }; + } +} + +#[cfg(not(test))] +unsafe fn generate_lorem(buffer: *mut c_void, size: usize) { + unsafe { LOREM_genBuffer(buffer, size, 0) }; +} + +#[cfg(test)] +unsafe fn generate_lorem(buffer: *mut c_void, size: usize) { + if size != 0 { + unsafe { ptr::write_bytes(buffer.cast::(), b'l', size) }; + } +} + +#[inline] +fn run_succeeded(outcome: BMK_runOutcome_t) -> bool { + outcome.error_tag_never_ever_use_directly == 0 +} + +#[inline] +fn c_speed(src_size: usize, nano_sec_per_run: f64) -> u64 { + if nano_sec_per_run <= 0.0 || !nano_sec_per_run.is_finite() { + return 0; + } + ((src_size as f64 * 1_000_000_000.0) / nano_sec_per_run) as u64 +} + +fn display_label(name: &str) -> String { + let name = display_name(name); + let bytes = name.as_bytes(); + if bytes.len() <= 17 { + return name.to_owned(); + } + String::from_utf8_lossy(&bytes[bytes.len() - 17..]).into_owned() +} + +fn ratio_digits(ratio: f64) -> usize { + 1 + usize::from(ratio < 100.0) + usize::from(ratio < 10.0) +} + +fn output_progress( + display_level: c_int, + mark: &str, + name: &str, + src_size: usize, + c_size: usize, + ratio: f64, + c_speed: u64, + d_speed: Option, +) { + if display_level < 2 { + return; + } + let label = display_label(name); + let ratio = format!("{:.*}", ratio_digits(ratio), ratio); + let c_speed_digits = usize::from(c_speed < 10 * MB_UNIT); + let c_speed = format!("{:.*}", c_speed_digits + 1, c_speed as f64 / MB_UNIT as f64); + let mut stdout = io::stdout(); + if let Some(d_speed) = d_speed { + let d_speed = format!("{d_speed:6.1}", d_speed = d_speed as f64 / MB_UNIT as f64); + let _ = write!( + stdout, + "{mark:>2}-{label:<17} :{src_size:>10} ->{c_size:>10} (x{ratio:>5}), {c_speed:>6} MB/s, {d_speed} MB/s\r" + ); + } else { + let _ = write!( + stdout, + "{mark:>2}-{label:<17} :{src_size:>10} ->{c_size:>10} (x{ratio:>5}), {c_speed:>6} MB/s\r" + ); + } + let _ = stdout.flush(); +} + +fn set_realtime_priority() { + #[cfg(unix)] + unsafe { + let _ = libc::setpriority(libc::PRIO_PROCESS, 0, -20); + } +} + +#[inline] +fn try_bytes(size: usize) -> Result, ()> { + let mut result = Vec::new(); + result.try_reserve_exact(size).map_err(|_| ())?; + result.resize(size, 0); + Ok(result) +} + +#[inline] +fn try_vec(capacity: usize) -> Result, ()> { + let mut result = Vec::new(); + result.try_reserve_exact(capacity).map_err(|_| ())?; + Ok(result) +} + +unsafe fn c_string_path(value: *const c_char) -> Option { + if value.is_null() { + return None; + } + let bytes = unsafe { CStr::from_ptr(value) }.to_bytes(); + #[cfg(unix)] + { + Some(PathBuf::from(std::ffi::OsStr::from_bytes(bytes))) + } + #[cfg(not(unix))] + { + Some(PathBuf::from(String::from_utf8_lossy(bytes).into_owned())) + } +} + +fn display_name(value: &str) -> &str { + value.rsplit(['/', '\\']).next().unwrap_or(value) +} + +fn zstd_error_callback(result: usize) -> c_uint { + c_uint::from(is_zstd_error(result)) +} + +unsafe extern "C" fn zstd_error_callback_abi(result: usize) -> c_uint { + zstd_error_callback(result) +} + +#[derive(Clone, Copy)] +struct BenchCodecContext { + cctx: *mut c_void, + dctx: *mut c_void, + dict: *const c_void, + dict_size: usize, + c_level: c_int, + cparams: ZSTD_compressionParameters, + adv: BMK_advancedParams_t, + init_error: usize, +} + +unsafe fn set_c_parameter(context: &mut BenchCodecContext, param: c_int, value: c_int) -> bool { + let result = unsafe { zstd_cctx_set_parameter(context.cctx, param, value) }; + if is_zstd_error(result) { + context.init_error = result; + false + } else { + true + } +} + +unsafe fn initialize_compression(context: &mut BenchCodecContext) -> usize { + context.init_error = 0; + let reset = unsafe { zstd_cctx_reset(context.cctx) }; + if is_zstd_error(reset) { + context.init_error = reset; + return reset; + } + + let workers = if context.adv.nbWorkers == 1 { + 0 + } else { + context.adv.nbWorkers + }; + let parameters = [ + (ZSTD_C_NB_WORKERS, workers), + (ZSTD_C_COMPRESSION_LEVEL, context.c_level), + (ZSTD_C_USE_ROW_MATCH_FINDER, context.adv.useRowMatchFinder), + (ZSTD_C_ENABLE_LDM, context.adv.ldmFlag), + (ZSTD_C_LDM_MIN_MATCH, context.adv.ldmMinMatch), + (ZSTD_C_LDM_HASH_LOG, context.adv.ldmHashLog), + (ZSTD_C_LDM_BUCKET_SIZE_LOG, context.adv.ldmBucketSizeLog), + (ZSTD_C_LDM_HASH_RATE_LOG, context.adv.ldmHashRateLog), + (ZSTD_C_WINDOW_LOG, context.cparams.windowLog as c_int), + (ZSTD_C_HASH_LOG, context.cparams.hashLog as c_int), + (ZSTD_C_CHAIN_LOG, context.cparams.chainLog as c_int), + (ZSTD_C_SEARCH_LOG, context.cparams.searchLog as c_int), + (ZSTD_C_MIN_MATCH, context.cparams.minMatch as c_int), + (ZSTD_C_TARGET_LENGTH, context.cparams.targetLength as c_int), + ( + ZSTD_C_LITERAL_COMPRESSION_MODE, + context.adv.literalCompressionMode, + ), + (ZSTD_C_STRATEGY, context.cparams.strategy), + ( + ZSTD_C_TARGET_C_BLOCK_SIZE, + context.adv.targetCBlockSize as c_int, + ), + ]; + for (param, value) in parameters { + if !unsafe { set_c_parameter(context, param, value) } { + return context.init_error; + } + } + + let result = + unsafe { zstd_cctx_load_dictionary(context.cctx, context.dict, context.dict_size) }; + if is_zstd_error(result) { + context.init_error = result; + } + result +} + +unsafe fn initialize_decompression(context: &mut BenchCodecContext) -> usize { + context.init_error = 0; + let reset = unsafe { zstd_dctx_reset(context.dctx) }; + if is_zstd_error(reset) { + context.init_error = reset; + return reset; + } + let result = + unsafe { zstd_dctx_load_dictionary(context.dctx, context.dict, context.dict_size) }; + if is_zstd_error(result) { + context.init_error = result; + } + result +} + +unsafe extern "C" fn initialize_compression_abi(payload: *mut c_void) -> usize { + let context = unsafe { &mut *payload.cast::() }; + unsafe { initialize_compression(context) } +} + +unsafe extern "C" fn initialize_decompression_abi(payload: *mut c_void) -> usize { + let context = unsafe { &mut *payload.cast::() }; + unsafe { initialize_decompression(context) } +} + +unsafe extern "C" fn compress_block_abi( + src: *const c_void, + src_size: usize, + dst: *mut c_void, + dst_capacity: usize, + payload: *mut c_void, +) -> usize { + let context = unsafe { &mut *payload.cast::() }; + if context.init_error != 0 { + return context.init_error; + } + unsafe { zstd_compress2(context.cctx, dst, dst_capacity, src, src_size) } +} + +unsafe extern "C" fn decompress_block_abi( + src: *const c_void, + src_size: usize, + dst: *mut c_void, + dst_capacity: usize, + payload: *mut c_void, +) -> usize { + let context = unsafe { &mut *payload.cast::() }; + if context.init_error != 0 { + return context.init_error; + } + + let mut input = ZSTD_inBuffer { + src, + size: src_size, + pos: 0, + }; + let mut output = ZSTD_outBuffer { + dst, + size: dst_capacity, + pos: 0, + }; + let mut remaining = 1usize; + while remaining != 0 { + if output.pos == output.size { + return error_value(ZSTD_ERROR_DST_SIZE_TOO_SMALL); + } + remaining = unsafe { zstd_decompress_stream(context.dctx, &mut output, &mut input) }; + if is_zstd_error(remaining) { + return remaining; + } + } + output.pos +} + +struct BlockSpec { + src_offset: usize, + src_size: usize, + result_size: usize, +} + +struct BenchBuffers { + src_ptrs: Vec<*const c_void>, + src_sizes: Vec, + c_ptrs: Vec<*mut c_void>, + c_capacities: Vec, + c_sizes: Vec, + result_ptrs: Vec<*mut c_void>, + result_capacities: Vec, + _compressed_storage: Option>, + result_storage: Vec, +} + +unsafe fn build_block_specs( + src: *const c_void, + src_size: usize, + file_sizes: &[usize], + mode: BMK_mode_t, + block_size: usize, +) -> Result<(Vec, usize), c_int> { + let mut specs = try_vec(file_sizes.len()).map_err(|_| 31)?; + let mut source_offset = 0usize; + let mut logical_size = src_size; + + if mode == BMK_decodeOnly { + logical_size = 0; + for &file_size in file_sizes { + if source_offset.checked_add(file_size).is_none() { + return Err(32); + } + let file_src = unsafe { (src.cast::()).add(source_offset) }.cast(); + let decoded = unsafe { zstd_find_decompressed_size(file_src, file_size) }; + if decoded == ZSTD_CONTENTSIZE_UNKNOWN || decoded == ZSTD_CONTENTSIZE_ERROR { + return Err(32); + } + let decoded = usize::try_from(decoded).map_err(|_| 32)?; + logical_size = logical_size.checked_add(decoded).ok_or(32)?; + specs.push(BlockSpec { + src_offset: source_offset, + src_size: file_size, + result_size: decoded, + }); + source_offset += file_size; + } + return Ok((specs, logical_size)); + } + + for &file_size in file_sizes { + let mut remaining = file_size; + while remaining != 0 { + let this_size = remaining.min(block_size); + specs.push(BlockSpec { + src_offset: source_offset, + src_size: this_size, + result_size: this_size, + }); + source_offset += this_size; + remaining -= this_size; + } + } + Ok((specs, logical_size)) +} + +unsafe fn build_bench_buffers( + src: *const c_void, + src_size: usize, + dst: *mut c_void, + dst_capacity: usize, + specs: &[BlockSpec], + mode: BMK_mode_t, +) -> Result { + let mut compressed_size = 0usize; + let mut compressed_capacities = try_vec(specs.len()).map_err(|_| 31)?; + for spec in specs { + let capacity = if mode == BMK_decodeOnly { + spec.src_size + } else { + let bound = unsafe { zstd_compress_bound(spec.src_size) }; + if is_zstd_error(bound) { + return Err(31); + } + bound + }; + compressed_size = compressed_size.checked_add(capacity).ok_or(31)?; + compressed_capacities.push(capacity); + } + + if (dst.is_null()) != (dst_capacity == 0) { + return Err(32); + } + if !dst.is_null() && dst_capacity < compressed_size { + return Err(32); + } + + let compressed_storage = if dst.is_null() { + let mut capacity = compressed_size; + if mode != BMK_decodeOnly { + capacity = capacity + .checked_add(specs.len().saturating_mul(1024)) + .ok_or(31)?; + } + Some(try_bytes(capacity).map_err(|_| 31)?) + } else { + None + }; + let compressed_base = if let Some(storage) = compressed_storage.as_ref() { + storage.as_ptr().cast_mut() + } else { + dst.cast::() + }; + + if mode == BMK_decodeOnly && src_size != 0 { + unsafe { ptr::copy_nonoverlapping(src.cast::(), compressed_base, src_size) }; + } + + let mut result_size = 0usize; + for spec in specs { + result_size = result_size.checked_add(spec.result_size).ok_or(31)?; + } + let result_storage = try_bytes(result_size).map_err(|_| 31)?; + let result_base = result_storage.as_ptr().cast_mut(); + + let mut src_ptrs = try_vec(specs.len()).map_err(|_| 31)?; + let mut src_sizes = try_vec(specs.len()).map_err(|_| 31)?; + let mut c_ptrs = try_vec(specs.len()).map_err(|_| 31)?; + let mut c_capacities = try_vec(specs.len()).map_err(|_| 31)?; + let mut c_sizes = try_vec(specs.len()).map_err(|_| 31)?; + let mut result_ptrs = try_vec(specs.len()).map_err(|_| 31)?; + let mut result_capacities = try_vec(specs.len()).map_err(|_| 31)?; + let mut c_offset = 0usize; + let mut result_offset = 0usize; + + for (index, spec) in specs.iter().enumerate() { + let src_ptr = unsafe { src.cast::().add(spec.src_offset) }.cast(); + let c_ptr = unsafe { compressed_base.add(c_offset) }.cast(); + let result_ptr = unsafe { result_base.add(result_offset) }.cast(); + src_ptrs.push(src_ptr); + src_sizes.push(spec.src_size); + c_ptrs.push(c_ptr); + c_capacities.push(compressed_capacities[index]); + c_sizes.push(if mode == BMK_decodeOnly { + spec.src_size + } else { + 0 + }); + result_ptrs.push(result_ptr); + result_capacities.push(spec.result_size); + c_offset += compressed_capacities[index]; + result_offset += spec.result_size; + } + + Ok(BenchBuffers { + src_ptrs, + src_sizes, + c_ptrs, + c_capacities, + c_sizes, + result_ptrs, + result_capacities, + _compressed_storage: compressed_storage, + result_storage, + }) +} + +unsafe fn bench_mem_advanced( + src: *const c_void, + src_size: usize, + dst: *mut c_void, + dst_capacity: usize, + file_sizes: &[usize], + c_level: c_int, + cparams: ZSTD_compressionParameters, + dict: *const c_void, + dict_size: usize, + display_level: c_int, + name: &str, + adv: BMK_advancedParams_t, +) -> BMK_benchOutcome_t { + if (dst.is_null()) != (dst_capacity == 0) { + return bench_error(32); + } + if file_sizes.is_empty() || src.is_null() && src_size != 0 { + return bench_error(32); + } + let file_total = file_sizes + .iter() + .try_fold(0usize, |total, size| total.checked_add(*size)); + if file_total != Some(src_size) { + return bench_error(32); + } + if dict_size != 0 && dict.is_null() { + return bench_error(32); + } + if src_size == 0 { + return bench_error(31); + } + + let block_size = if adv.mode != BMK_decodeOnly && adv.blockSize >= 32 { + adv.blockSize + } else { + src_size + }; + let (specs, logical_size) = match unsafe { + build_block_specs(src, src_size, file_sizes, adv.mode, block_size.max(1)) + } { + Ok(value) => value, + Err(code) => return bench_error(code), + }; + if specs.is_empty() { + return bench_error(31); + } + let mut buffers = + match unsafe { build_bench_buffers(src, src_size, dst, dst_capacity, &specs, adv.mode) } { + Ok(value) => value, + Err(code) => return bench_error(code), + }; + + let cctx = unsafe { zstd_create_cctx() }; + let dctx = unsafe { zstd_create_dctx() }; + if cctx.is_null() || dctx.is_null() { + unsafe { + zstd_free_cctx(cctx); + zstd_free_dctx(dctx); + } + return bench_error(31); + } + + let total_ms = adv.nbSeconds.wrapping_mul(1_000); + let compression_state = unsafe { timed_state_create(total_ms) }; + let decompression_state = unsafe { timed_state_create(total_ms) }; + if compression_state.is_null() || decompression_state.is_null() { + unsafe { + timed_state_free(compression_state); + timed_state_free(decompression_state); + zstd_free_cctx(cctx); + zstd_free_dctx(dctx); + } + return bench_error(31); + } + + let mut context = BenchCodecContext { + cctx, + dctx, + dict, + dict_size, + c_level, + cparams, + adv, + init_error: 0, + }; + let payload = (&mut context as *mut BenchCodecContext).cast::(); + let mut compression_params = BMK_benchParams_t { + benchFn: Some(compress_block_abi), + benchPayload: payload, + initFn: Some(initialize_compression_abi), + initPayload: payload, + errorFn: Some(zstd_error_callback_abi), + blockCount: buffers.src_ptrs.len(), + srcBuffers: buffers.src_ptrs.as_ptr(), + srcSizes: buffers.src_sizes.as_ptr(), + dstBuffers: buffers.c_ptrs.as_ptr(), + dstCapacities: buffers.c_capacities.as_ptr(), + blockResults: buffers.c_sizes.as_mut_ptr(), + }; + let decompression_params = BMK_benchParams_t { + benchFn: Some(decompress_block_abi), + benchPayload: payload, + initFn: Some(initialize_decompression_abi), + initPayload: payload, + errorFn: Some(zstd_error_callback_abi), + blockCount: buffers.c_ptrs.len(), + srcBuffers: buffers.c_ptrs.as_ptr().cast(), + srcSizes: buffers.c_sizes.as_ptr(), + dstBuffers: buffers.result_ptrs.as_ptr(), + dstCapacities: buffers.result_capacities.as_ptr(), + blockResults: ptr::null_mut(), + }; + + let mut compression_done = adv.mode == BMK_decodeOnly; + let mut decompression_done = adv.mode == BMK_compressOnly; + let mut result = BMK_benchResult_t::default(); + let mut c_size = if adv.mode == BMK_decodeOnly { + src_size + } else { + 0 + }; + let mut ratio = if c_size == 0 { + 0.0 + } else { + logical_size as f64 / c_size as f64 + }; + + let marks = [" |", " /", " =", " \\"]; + let mut mark_number = 0usize; + if display_level >= 2 { + let mut stdout = io::stdout(); + let _ = write!(stdout, "\r{:>70}\r", ""); + let _ = stdout.flush(); + output_progress( + display_level, + marks[mark_number], + name, + logical_size, + c_size, + ratio, + result.cSpeed, + None, + ); + } + + while !(compression_done && decompression_done) { + if !compression_done { + let outcome = unsafe { timed_state_run(compression_state, compression_params) }; + if !run_succeeded(outcome) { + unsafe { + timed_state_free(compression_state); + timed_state_free(decompression_state); + zstd_free_cctx(cctx); + zstd_free_dctx(dctx); + } + return bench_error(30); + } + let runtime = outcome.internal_never_ever_use_directly; + c_size = runtime.sumOfReturn; + ratio = if c_size == 0 { + 0.0 + } else { + logical_size as f64 / c_size as f64 + }; + result.cSize = c_size; + result.cSpeed = result + .cSpeed + .max(c_speed(logical_size, runtime.nanoSecPerRun)); + output_progress( + display_level, + marks[mark_number], + name, + logical_size, + c_size, + ratio, + result.cSpeed, + None, + ); + compression_done = unsafe { timed_state_complete(compression_state) }; + } + + if !decompression_done { + let outcome = unsafe { timed_state_run(decompression_state, decompression_params) }; + if !run_succeeded(outcome) { + unsafe { + timed_state_free(compression_state); + timed_state_free(decompression_state); + zstd_free_cctx(cctx); + zstd_free_dctx(dctx); + } + return bench_error(30); + } + let runtime = outcome.internal_never_ever_use_directly; + result.dSpeed = result + .dSpeed + .max(c_speed(logical_size, runtime.nanoSecPerRun)); + output_progress( + display_level, + marks[mark_number], + name, + logical_size, + c_size, + ratio, + result.cSpeed, + Some(result.dSpeed), + ); + decompression_done = unsafe { timed_state_complete(decompression_state) }; + } + mark_number = (mark_number + 1) % marks.len(); + } + + if adv.mode == BMK_both { + let result_bytes = &buffers.result_storage; + let source_bytes = unsafe { std::slice::from_raw_parts(src.cast::(), src_size) }; + if result_bytes != source_bytes && display_level >= 1 { + eprintln!( + "!!! WARNING !!! {:>17} : Invalid Checksum", + display_name(name) + ); + } + } + + if display_level == 1 { + let c_speed = result.cSpeed as f64 / MB_UNIT as f64; + let d_speed = result.dSpeed as f64 / MB_UNIT as f64; + if adv.additionalParam != 0 { + println!( + "-{c_level:<3}{c_size:>11} ({ratio:5.3}) {c_speed:6.2} MB/s {d_speed:6.1} MB/s {} (param={})", + display_name(name), adv.additionalParam + ); + } else { + println!( + "-{c_level:<3}{c_size:>11} ({ratio:5.3}) {c_speed:6.2} MB/s {d_speed:6.1} MB/s {}", + display_name(name) + ); + } + } else if display_level >= 2 { + println!("{c_level:2}#"); + } + + let window_memory = if cparams.windowLog < usize::BITS { + 1usize << cparams.windowLog + } else { + 0 + }; + result.cMem = window_memory.saturating_add(unsafe { zstd_sizeof_cctx(cctx) }); + + unsafe { + timed_state_free(compression_state); + timed_state_free(decompression_state); + zstd_free_cctx(cctx); + zstd_free_dctx(dctx); + } + /* Keep buffers alive until every callback and context has finished. */ + let _ = (&mut buffers, &mut compression_params, ratio); + result_outcome(result) +} + +fn bench_error(_tag: c_int) -> BMK_benchOutcome_t { + BMK_benchOutcome_t { + internal_never_ever_use_directly: BMK_benchResult_t::default(), + tag: 1, + } +} + +fn result_outcome(result: BMK_benchResult_t) -> BMK_benchOutcome_t { + BMK_benchOutcome_t { + internal_never_ever_use_directly: result, + tag: 0, + } +} + +unsafe fn bench_c_levels( + src: *const c_void, + src_size: usize, + file_sizes: &[usize], + start_level: c_int, + end_level: c_int, + cparams: ZSTD_compressionParameters, + dict: *const c_void, + dict_size: usize, + display_level: c_int, + name: &str, + adv: BMK_advancedParams_t, +) -> c_int { + if end_level > unsafe { zstd_max_c_level() } || end_level < start_level { + if display_level >= 1 { + eprintln!( + "{}", + if end_level < start_level { + "Invalid Compression Level Range" + } else { + "Invalid Compression Level" + } + ); + } + return 15; + } + if adv.realTime != 0 { + if display_level >= 2 { + eprintln!("Note : switching to real-time priority"); + } + set_realtime_priority(); + } + if display_level == 1 && adv.additionalParam == 0 { + let version = unsafe { zstd_version() }; + println!( + "bench {version} : input {} bytes, {} seconds, {} KB blocks", + src_size, + adv.nbSeconds, + adv.blockSize >> 10 + ); + } + + let mut level = start_level; + loop { + let outcome = unsafe { + bench_mem_advanced( + src, + src_size, + ptr::null_mut(), + 0, + file_sizes, + level, + cparams, + dict, + dict_size, + display_level, + name, + adv, + ) + }; + if BMK_isSuccessful_benchOutcome(outcome) == 0 { + return 1; + } + if level == end_level { + break; + } + level = level.saturating_add(1); + } + 0 +} + +fn max_memory() -> u64 { + if usize::BITS == 32 { + (2u64 << 30) - (64u64 << 20) + } else { + 1u64 << (usize::BITS - 31) + } +} + +fn find_max_memory(required_memory: u64) -> usize { + let step = 64u64 << 20; + let mut required = ((required_memory >> 26) + 1) << 26; + required = required.saturating_add(step).min(max_memory()); + + loop { + let allocation = usize::try_from(required) + .ok() + .map(|size| unsafe { libc::malloc(size) }); + required = required.saturating_sub(step); + if let Some(allocation) = allocation { + if !allocation.is_null() { + unsafe { libc::free(allocation) }; + return required as usize; + } + } + if required == 0 { + return 0; + } + } +} + +fn load_files( + buffer: &mut [u8], + file_sizes: &mut [usize], + paths: &[PathBuf], + display_level: c_int, +) -> Result<(), c_int> { + let mut position = 0usize; + let mut total_size = 0usize; + + for (index, path) in paths.iter().enumerate() { + let metadata = match fs::metadata(path) { + Ok(metadata) => metadata, + Err(_) => return Err(1), + }; + if metadata.is_dir() { + if display_level >= 2 { + println!("Ignoring {} directory... ", path.display()); + } + file_sizes[index] = 0; + continue; + } + if !metadata.is_file() { + file_sizes[index] = 0; + continue; + } + + let mut file_size = usize::try_from(metadata.len()).map_err(|_| 1)?; + let available = buffer.len().saturating_sub(position); + let truncated = file_size > available; + if truncated { + file_size = available; + } + + if display_level >= 2 { + print!("Loading {}... \r", path.display()); + let _ = io::stdout().flush(); + } + let mut file = fs::File::open(path).map_err(|_| 1)?; + file.read_exact(&mut buffer[position..position + file_size]) + .map_err(|_| 1)?; + file_sizes[index] = file_size; + position += file_size; + total_size += file_size; + + if truncated { + break; + } + } + + if total_size == 0 { + return Err(12); + } + Ok(()) +} + +unsafe fn collect_file_inputs( + file_names: *const *const c_char, + nb_files: c_uint, + display_level: c_int, +) -> Result<(Vec, Vec, String), c_int> { + if file_names.is_null() || nb_files == 0 { + return Err(13); + } + let mut paths = try_vec(nb_files as usize).map_err(|_| 16)?; + for index in 0..nb_files as usize { + let path = unsafe { c_string_path(*file_names.add(index)) }.ok_or(15)?; + paths.push(path); + } + + let mut total_size = 0u64; + let mut sizes = try_vec(paths.len()).map_err(|_| 16)?; + for path in &paths { + let metadata = fs::metadata(path).map_err(|_| 15)?; + if !metadata.is_file() { + return Err(15); + } + total_size = total_size.checked_add(metadata.len()).ok_or(15)?; + sizes.push(usize::try_from(metadata.len()).map_err(|_| 15)?); + } + if total_size == 0 { + return Err(1); + } + let benched_size = (find_max_memory(total_size.saturating_mul(3)) / 3) + .min(usize::try_from(total_size).map_err(|_| 20)?); + if benched_size == 0 { + return Err(20); + } + if u64::try_from(benched_size).unwrap_or(u64::MAX) < total_size { + eprintln!( + "Not enough memory; testing {} MB only...", + benched_size >> 20 + ); + } + let mut source = try_bytes(benched_size).map_err(|_| 20)?; + sizes.fill(0); + match load_files(&mut source, &mut sizes, &paths, display_level) { + Ok(()) => {} + Err(12) => return Err(1), + Err(_) => return Err(1), + } + let name = if nb_files > 1 { + format!(" {nb_files} files") + } else { + unsafe { CStr::from_ptr(*file_names) } + .to_string_lossy() + .into_owned() + }; + Ok((source, sizes, name)) +} + +unsafe fn collect_dictionary( + dict_file_name: *const c_char, + display_level: c_int, +) -> Result, c_int> { + let Some(path) = (unsafe { c_string_path(dict_file_name) }) else { + return Ok(Vec::new()); + }; + let metadata = fs::metadata(&path).map_err(|_| 17)?; + if !metadata.is_file() { + return Err(17); + } + let size = usize::try_from(metadata.len()).map_err(|_| 18)?; + if size > 64 * 1024 * 1024 { + return Err(18); + } + let mut data = try_bytes(size).map_err(|_| 19)?; + let mut file_sizes = [0usize; 1]; + match load_files( + data.as_mut_slice(), + &mut file_sizes, + std::slice::from_ref(&path), + display_level, + ) { + Ok(()) => {} + Err(_) => return Err(1), + } + Ok(data) +} + +/// Returns the defaults used by the non-advanced benchmark entry points. +#[no_mangle] +pub extern "C" fn BMK_initAdvancedParams() -> BMK_advancedParams_t { + BMK_advancedParams_t::default() +} + +/// Returns nonzero when a benchmark outcome contains a valid result. +#[no_mangle] +pub extern "C" fn BMK_isSuccessful_benchOutcome(outcome: BMK_benchOutcome_t) -> c_int { + c_int::from(outcome.tag == 0) +} + +/// Extracts a valid result and aborts when called on an error outcome. +#[no_mangle] +pub extern "C" fn BMK_extract_benchResult(outcome: BMK_benchOutcome_t) -> BMK_benchResult_t { + if outcome.tag != 0 { + std::process::abort(); + } + outcome.internal_never_ever_use_directly +} + +/// Core memory-to-memory benchmark entry point from `benchzstd.h`. +#[no_mangle] +pub unsafe extern "C" fn BMK_benchMemAdvanced( + srcBuffer: *const c_void, + srcSize: usize, + dstBuffer: *mut c_void, + dstCapacity: usize, + fileSizes: *const usize, + nbFiles: c_uint, + cLevel: c_int, + comprParams: *const ZSTD_compressionParameters, + dictBuffer: *const c_void, + dictBufferSize: usize, + displayLevel: c_int, + displayName: *const c_char, + adv: *const BMK_advancedParams_t, +) -> BMK_benchOutcome_t { + if fileSizes.is_null() || comprParams.is_null() || adv.is_null() { + return bench_error(32); + } + let file_sizes = unsafe { std::slice::from_raw_parts(fileSizes, nbFiles as usize) }; + let name = if displayName.is_null() { + String::new() + } else { + unsafe { CStr::from_ptr(displayName) } + .to_string_lossy() + .into_owned() + }; + unsafe { + bench_mem_advanced( + srcBuffer, + srcSize, + dstBuffer, + dstCapacity, + file_sizes, + cLevel, + *comprParams, + dictBuffer, + dictBufferSize, + displayLevel, + &name, + *adv, + ) + } +} + +/// Default-parameter memory-to-memory benchmark entry point. +#[no_mangle] +pub unsafe extern "C" fn BMK_benchMem( + srcBuffer: *const c_void, + srcSize: usize, + fileSizes: *const usize, + nbFiles: c_uint, + cLevel: c_int, + comprParams: *const ZSTD_compressionParameters, + dictBuffer: *const c_void, + dictBufferSize: usize, + displayLevel: c_int, + displayName: *const c_char, +) -> BMK_benchOutcome_t { + let adv = BMK_initAdvancedParams(); + unsafe { + BMK_benchMemAdvanced( + srcBuffer, + srcSize, + ptr::null_mut(), + 0, + fileSizes, + nbFiles, + cLevel, + comprParams, + dictBuffer, + dictBufferSize, + displayLevel, + displayName, + &adv, + ) + } +} + +/// Loads files, optionally loads a dictionary, and benchmarks every level in +/// the requested range. +#[no_mangle] +pub unsafe extern "C" fn BMK_benchFilesAdvanced( + fileNamesTable: *const *const c_char, + nbFiles: c_uint, + dictFileName: *const c_char, + startCLevel: c_int, + endCLevel: c_int, + compressionParams: *const ZSTD_compressionParameters, + displayLevel: c_int, + adv: *const BMK_advancedParams_t, +) -> c_int { + if nbFiles == 0 { + return 13; + } + if compressionParams.is_null() || adv.is_null() { + return 15; + } + if endCLevel > unsafe { zstd_max_c_level() } { + return 14; + } + let (source, file_sizes, name) = + match unsafe { collect_file_inputs(fileNamesTable, nbFiles, displayLevel) } { + Ok(value) => value, + Err(code) => return code, + }; + let dictionary = match unsafe { collect_dictionary(dictFileName, displayLevel) } { + Ok(value) => value, + Err(code) => return code, + }; + let dict_ptr = if dictionary.is_empty() { + ptr::null() + } else { + dictionary.as_ptr().cast() + }; + unsafe { + bench_c_levels( + source.as_ptr().cast(), + source.len(), + &file_sizes, + startCLevel, + endCLevel, + *compressionParams, + dict_ptr, + dictionary.len(), + displayLevel, + &name, + *adv, + ) + } +} + +/// Single-level convenience wrapper used by older callers. +#[no_mangle] +pub unsafe extern "C" fn BMK_benchFiles( + fileNamesTable: *const *const c_char, + nbFiles: c_uint, + dictFileName: *const c_char, + cLevel: c_int, + compressionParams: *const ZSTD_compressionParameters, + displayLevel: c_int, +) -> c_int { + let adv = BMK_initAdvancedParams(); + unsafe { + BMK_benchFilesAdvanced( + fileNamesTable, + nbFiles, + dictFileName, + cLevel, + cLevel, + compressionParams, + displayLevel, + &adv, + ) + } +} + +/// Generates a deterministic synthetic sample and benchmarks the requested +/// level range. Negative compressibility selects the lorem ipsum generator, +/// matching `programs/benchzstd.c`. +#[no_mangle] +pub unsafe extern "C" fn BMK_syntheticTest( + compressibility: f64, + startingCLevel: c_int, + endCLevel: c_int, + compressionParams: *const ZSTD_compressionParameters, + displayLevel: c_int, + adv: *const BMK_advancedParams_t, +) -> c_int { + if compressionParams.is_null() || adv.is_null() { + return 15; + } + let adv_value = unsafe { *adv }; + let size = if adv_value.blockSize == 0 { + 10_000_000 + } else { + adv_value.blockSize + }; + let mut source = match try_bytes(size) { + Ok(value) => value, + Err(()) => return 16, + }; + let name = if compressibility < 0.0 { + unsafe { generate_lorem(source.as_mut_ptr().cast(), source.len()) }; + "Lorem ipsum".to_owned() + } else { + unsafe { generate_rdg(source.as_mut_ptr().cast(), source.len(), compressibility) }; + format!("Synthetic {}%", (compressibility * 100.0) as u32) + }; + let file_sizes = [source.len()]; + unsafe { + bench_c_levels( + source.as_ptr().cast(), + source.len(), + &file_sizes, + startingCLevel, + endCLevel, + *compressionParams, + ptr::null(), + 0, + displayLevel, + &name, + adv_value, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn advanced_defaults_match_benchzstd_c() { + let defaults = BMK_initAdvancedParams(); + assert_eq!(defaults.mode, BMK_both); + assert_eq!(defaults.nbSeconds, BMK_TIMETEST_DEFAULT_S); + assert_eq!(defaults.blockSize, 0); + assert_eq!(defaults.targetCBlockSize, 0); + assert_eq!(defaults.nbWorkers, 0); + assert_eq!(defaults.realTime, 0); + assert_eq!(defaults.additionalParam, 0); + assert_eq!(defaults.ldmFlag, 0); + assert_eq!(defaults.ldmMinMatch, 0); + assert_eq!(defaults.ldmHashLog, 0); + assert_eq!(defaults.ldmBucketSizeLog, 0); + assert_eq!(defaults.ldmHashRateLog, 0); + assert_eq!(defaults.literalCompressionMode, ZSTD_ps_auto); + assert_eq!(defaults.useRowMatchFinder, 0); + } + + #[test] + fn outcome_accessors_extract_only_valid_results() { + let result = BMK_benchResult_t { + cSize: 123, + cSpeed: 456, + dSpeed: 789, + cMem: 321, + }; + let valid = result_outcome(result); + assert_eq!(BMK_isSuccessful_benchOutcome(valid), 1); + assert_eq!(BMK_extract_benchResult(valid), result); + + let invalid = bench_error(30); + assert_eq!(BMK_isSuccessful_benchOutcome(invalid), 0); + assert_eq!(invalid.tag, 1); + } + + #[test] + fn outcome_layout_matches_header_variant() { + assert_eq!(size_of::(), size_of::() * 4); + assert_eq!( + offset_of!(BMK_benchOutcome_t, tag), + size_of::() + ); + } + + #[test] + fn public_parameter_layout_matches_the_c_headers() { + assert_eq!( + offset_of!(ZSTD_compressionParameters, strategy), + 6 * size_of::() + ); + assert_eq!( + offset_of!(BMK_advancedParams_t, targetCBlockSize), + 2 * size_of::() + size_of::() + ); + } + + #[test] + fn block_partitioning_preserves_file_boundaries() { + let (specs, logical_size) = unsafe { + build_block_specs(ptr::null(), 10, &[3, 7], BMK_both, 4).expect("valid block partition") + }; + assert_eq!(logical_size, 10); + assert_eq!( + specs + .iter() + .map(|spec| (spec.src_offset, spec.src_size)) + .collect::>(), + vec![(0, 3), (3, 4), (7, 3)] + ); + } + + #[test] + fn progress_labels_match_c_basename_and_tail_rules() { + assert_eq!(display_label("/tmp/input"), "input"); + assert_eq!( + display_label("/tmp/01234567890123456789"), + "34567890123456789" + ); + } +}