zstd-rs/rust/src/benchzstd.rs

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(non_upper_case_globals)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]

//! Benchmark orchestration for the command-line programs.
//!
//! This is the Rust counterpart of `programs/benchzstd.c`.  The public
//! structures intentionally mirror `programs/benchzstd.h` instead of using
//! Rust-only enums or references: they cross the C ABI by value or through
//! raw pointers.  The actual codec calls use the public zstd context API, so
//! this module does not depend on the private `ZSTD_CCtx_s` or `ZSTD_DCtx_s`
//! layouts.
//!
//! The module is kept independent of the CLI dispatcher.  The full CLI archive
//! includes it and the program build omits `benchzstd.c`; the existing
//! `benchfn.rs`, `datagen.rs`, and `lorem.rs` symbols are used through their
//! stable C ABI in that configuration.

use std::ffi::{c_char, c_void, CStr};
use std::fs;
use std::io::{self, Read, Write};
use std::mem::{offset_of, size_of};
use std::os::raw::{c_int, c_uint};
use std::path::PathBuf;
use std::ptr;

#[cfg(unix)]
use std::os::unix::ffi::OsStrExt;

pub const MB_UNIT: u64 = 1_000_000;
pub const BMK_TIMETEST_DEFAULT_S: c_uint = 3;
#[cfg(not(test))]
const BMK_RUNTEST_DEFAULT_MS: c_uint = 1_000;
#[cfg(test)]
const ZSTD_MAX_CLEVEL_FALLBACK: c_int = 22;
const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
const ZSTD_CONTENTSIZE_ERROR: u64 = u64::MAX - 1;
#[cfg(not(test))]
const ZSTD_RESET_SESSION_AND_PARAMETERS: c_int = 3;
const ZSTD_MAX_ERROR_CODE: c_int = 120;
#[cfg(test)]
const ZSTD_ERROR_GENERIC: c_int = 1;
const ZSTD_ERROR_DST_SIZE_TOO_SMALL: c_int = 70;

/* Public and experimental parameter values from zstd.h. */
const ZSTD_C_COMPRESSION_LEVEL: c_int = 100;
const ZSTD_C_WINDOW_LOG: c_int = 101;
const ZSTD_C_HASH_LOG: c_int = 102;
const ZSTD_C_CHAIN_LOG: c_int = 103;
const ZSTD_C_SEARCH_LOG: c_int = 104;
const ZSTD_C_MIN_MATCH: c_int = 105;
const ZSTD_C_TARGET_LENGTH: c_int = 106;
const ZSTD_C_STRATEGY: c_int = 107;
const ZSTD_C_TARGET_C_BLOCK_SIZE: c_int = 130;
const ZSTD_C_NB_WORKERS: c_int = 400;
const ZSTD_C_ENABLE_LDM: c_int = 160;
const ZSTD_C_LDM_HASH_LOG: c_int = 161;
const ZSTD_C_LDM_MIN_MATCH: c_int = 162;
const ZSTD_C_LDM_BUCKET_SIZE_LOG: c_int = 163;
const ZSTD_C_LDM_HASH_RATE_LOG: c_int = 164;
const ZSTD_C_LITERAL_COMPRESSION_MODE: c_int = 1002;
const ZSTD_C_USE_ROW_MATCH_FINDER: c_int = 1011;

/// `BMK_mode_t` is a C enum and therefore has the ABI of `int` here.
pub type BMK_mode_t = c_int;
pub const BMK_both: BMK_mode_t = 0;
pub const BMK_decodeOnly: BMK_mode_t = 1;
pub const BMK_compressOnly: BMK_mode_t = 2;

/// `ZSTD_ParamSwitch_e` is represented as an `int` in the C ABI.
pub type ZSTD_ParamSwitch_e = c_int;
pub const ZSTD_ps_auto: ZSTD_ParamSwitch_e = 0;
#[allow(dead_code)]
pub const ZSTD_ps_enable: ZSTD_ParamSwitch_e = 1;
#[allow(dead_code)]
pub const ZSTD_ps_disable: ZSTD_ParamSwitch_e = 2;

/// ABI-compatible `ZSTD_compressionParameters` from `zstd.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTD_compressionParameters {
    pub windowLog: u32,
    pub chainLog: u32,
    pub hashLog: u32,
    pub searchLog: u32,
    pub minMatch: u32,
    pub targetLength: u32,
    pub strategy: c_int,
}

/// ABI-compatible `BMK_benchResult_t` from `benchzstd.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct BMK_benchResult_t {
    pub cSize: usize,
    pub cSpeed: u64,
    pub dSpeed: u64,
    pub cMem: usize,
}

/// Opaque result variant from `benchzstd.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct BMK_benchOutcome_t {
    pub internal_never_ever_use_directly: BMK_benchResult_t,
    pub tag: c_int,
}

/// ABI-compatible advanced benchmark controls.
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BMK_advancedParams_t {
    pub mode: BMK_mode_t,
    pub nbSeconds: c_uint,
    pub blockSize: usize,
    pub targetCBlockSize: usize,
    pub nbWorkers: c_int,
    pub realTime: c_uint,
    pub additionalParam: c_int,
    pub ldmFlag: c_int,
    pub ldmMinMatch: c_int,
    pub ldmHashLog: c_int,
    pub ldmBucketSizeLog: c_int,
    pub ldmHashRateLog: c_int,
    pub literalCompressionMode: ZSTD_ParamSwitch_e,
    pub useRowMatchFinder: c_int,
}

impl Default for BMK_advancedParams_t {
    fn default() -> Self {
        Self {
            mode: BMK_both,
            nbSeconds: BMK_TIMETEST_DEFAULT_S,
            blockSize: 0,
            targetCBlockSize: 0,
            nbWorkers: 0,
            realTime: 0,
            additionalParam: 0,
            ldmFlag: 0,
            ldmMinMatch: 0,
            ldmHashLog: 0,
            ldmBucketSizeLog: 0,
            ldmHashRateLog: 0,
            literalCompressionMode: ZSTD_ps_auto,
            useRowMatchFinder: 0,
        }
    }
}

/* Keep the by-value C ABI honest on both 32- and 64-bit targets. */
const _: () = assert!(offset_of!(BMK_benchResult_t, cSpeed) == size_of::<usize>());
const _: () =
    assert!(offset_of!(BMK_benchResult_t, dSpeed) == size_of::<usize>() + size_of::<u64>());
const _: () = assert!(offset_of!(BMK_benchOutcome_t, tag) == size_of::<BMK_benchResult_t>());
const _: () = assert!(size_of::<ZSTD_compressionParameters>() == 7 * size_of::<u32>());
const _: () = assert!(offset_of!(ZSTD_compressionParameters, strategy) == 6 * size_of::<u32>());
const _: () = assert!(offset_of!(BMK_advancedParams_t, nbSeconds) == size_of::<c_int>());
const _: () = assert!(offset_of!(BMK_advancedParams_t, blockSize) == 2 * size_of::<c_int>());
const _: () = assert!(
    offset_of!(BMK_advancedParams_t, targetCBlockSize)
        == 2 * size_of::<c_int>() + size_of::<usize>()
);

/* The following mirrors are private to this module and are used only to
 * call the already migrated benchfn implementation by value. */
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
struct BMK_runTime_t {
    nanoSecPerRun: f64,
    sumOfReturn: usize,
}

#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
struct BMK_runOutcome_t {
    internal_never_ever_use_directly: BMK_runTime_t,
    error_result_never_ever_use_directly: usize,
    error_tag_never_ever_use_directly: c_int,
}

type BMK_benchFn_t =
    Option<unsafe extern "C" fn(*const c_void, usize, *mut c_void, usize, *mut c_void) -> usize>;
type BMK_initFn_t = Option<unsafe extern "C" fn(*mut c_void) -> usize>;
type BMK_errorFn_t = Option<unsafe extern "C" fn(usize) -> c_uint>;

#[repr(C)]
#[derive(Clone, Copy)]
struct BMK_benchParams_t {
    benchFn: BMK_benchFn_t,
    benchPayload: *mut c_void,
    initFn: BMK_initFn_t,
    initPayload: *mut c_void,
    errorFn: BMK_errorFn_t,
    blockCount: usize,
    srcBuffers: *const *const c_void,
    srcSizes: *const usize,
    dstBuffers: *const *mut c_void,
    dstCapacities: *const usize,
    blockResults: *mut usize,
}

#[repr(C)]
struct BMK_timedFnState_t {
    _private: [u8; 0],
}

#[repr(C)]
#[derive(Clone, Copy)]
struct ZSTD_inBuffer {
    src: *const c_void,
    size: usize,
    pos: usize,
}

#[repr(C)]
#[derive(Clone, Copy)]
struct ZSTD_outBuffer {
    dst: *mut c_void,
    size: usize,
    pos: usize,
}

#[cfg(not(test))]
unsafe extern "C" {
    fn ZSTD_maxCLevel() -> c_int;
    fn ZSTD_compressBound(srcSize: usize) -> usize;
    fn ZSTD_findDecompressedSize(src: *const c_void, srcSize: usize) -> u64;
    fn ZSTD_createCCtx() -> *mut c_void;
    fn ZSTD_freeCCtx(cctx: *mut c_void) -> usize;
    fn ZSTD_CCtx_reset(cctx: *mut c_void, reset: c_int) -> usize;
    fn ZSTD_CCtx_setParameter(cctx: *mut c_void, param: c_int, value: c_int) -> usize;
    fn ZSTD_CCtx_loadDictionary(cctx: *mut c_void, dict: *const c_void, dictSize: usize) -> usize;
    fn ZSTD_compress2(
        cctx: *mut c_void,
        dst: *mut c_void,
        dstCapacity: usize,
        src: *const c_void,
        srcSize: usize,
    ) -> usize;
    fn ZSTD_sizeof_CCtx(cctx: *const c_void) -> usize;
    fn ZSTD_createDCtx() -> *mut c_void;
    fn ZSTD_freeDCtx(dctx: *mut c_void) -> usize;
    fn ZSTD_DCtx_reset(dctx: *mut c_void, reset: c_int) -> usize;
    fn ZSTD_DCtx_loadDictionary(dctx: *mut c_void, dict: *const c_void, dictSize: usize) -> usize;
    fn ZSTD_decompressStream(
        dctx: *mut c_void,
        output: *mut ZSTD_outBuffer,
        input: *mut ZSTD_inBuffer,
    ) -> usize;
    fn ZSTD_versionString() -> *const c_char;

    fn BMK_createTimedFnState(total_ms: c_uint, run_ms: c_uint) -> *mut BMK_timedFnState_t;
    fn BMK_freeTimedFnState(state: *mut BMK_timedFnState_t);
    fn BMK_benchTimedFn(
        state: *mut BMK_timedFnState_t,
        params: BMK_benchParams_t,
    ) -> BMK_runOutcome_t;
    fn BMK_isCompleted_TimedFn(state: *const BMK_timedFnState_t) -> c_int;

    fn RDG_genBuffer(
        buffer: *mut c_void,
        size: usize,
        matchProba: f64,
        litProba: f64,
        seed: c_uint,
    );
    fn LOREM_genBuffer(buffer: *mut c_void, size: usize, seed: c_uint);
}

#[inline]
fn error_value(code: c_int) -> usize {
    (0usize).wrapping_sub(code as usize)
}

#[inline]
fn is_zstd_error(result: usize) -> bool {
    result > error_value(ZSTD_MAX_ERROR_CODE)
}

#[cfg(not(test))]
unsafe fn zstd_max_c_level() -> c_int {
    unsafe { ZSTD_maxCLevel() }
}

#[cfg(test)]
unsafe fn zstd_max_c_level() -> c_int {
    ZSTD_MAX_CLEVEL_FALLBACK
}

#[cfg(not(test))]
unsafe fn zstd_compress_bound(size: usize) -> usize {
    unsafe { ZSTD_compressBound(size) }
}

#[cfg(test)]
unsafe fn zstd_compress_bound(size: usize) -> usize {
    size.saturating_add(64)
}

#[cfg(not(test))]
unsafe fn zstd_find_decompressed_size(src: *const c_void, size: usize) -> u64 {
    unsafe { ZSTD_findDecompressedSize(src, size) }
}

#[cfg(test)]
unsafe fn zstd_find_decompressed_size(_src: *const c_void, _size: usize) -> u64 {
    ZSTD_CONTENTSIZE_ERROR
}

#[cfg(not(test))]
unsafe fn zstd_create_cctx() -> *mut c_void {
    unsafe { ZSTD_createCCtx() }
}

#[cfg(test)]
unsafe fn zstd_create_cctx() -> *mut c_void {
    ptr::dangling_mut::<c_void>()
}

#[cfg(not(test))]
unsafe fn zstd_free_cctx(cctx: *mut c_void) {
    unsafe { ZSTD_freeCCtx(cctx) };
}

#[cfg(test)]
unsafe fn zstd_free_cctx(_cctx: *mut c_void) {}

#[cfg(not(test))]
unsafe fn zstd_create_dctx() -> *mut c_void {
    unsafe { ZSTD_createDCtx() }
}

#[cfg(test)]
unsafe fn zstd_create_dctx() -> *mut c_void {
    ptr::dangling_mut::<c_void>()
}

#[cfg(not(test))]
unsafe fn zstd_free_dctx(dctx: *mut c_void) {
    unsafe { ZSTD_freeDCtx(dctx) };
}

#[cfg(test)]
unsafe fn zstd_free_dctx(_dctx: *mut c_void) {}

#[cfg(not(test))]
unsafe fn zstd_cctx_reset(cctx: *mut c_void) -> usize {
    unsafe { ZSTD_CCtx_reset(cctx, ZSTD_RESET_SESSION_AND_PARAMETERS) }
}

#[cfg(test)]
unsafe fn zstd_cctx_reset(_cctx: *mut c_void) -> usize {
    0
}

#[cfg(not(test))]
unsafe fn zstd_cctx_set_parameter(cctx: *mut c_void, param: c_int, value: c_int) -> usize {
    unsafe { ZSTD_CCtx_setParameter(cctx, param, value) }
}

#[cfg(test)]
unsafe fn zstd_cctx_set_parameter(_cctx: *mut c_void, _param: c_int, _value: c_int) -> usize {
    0
}

#[cfg(not(test))]
unsafe fn zstd_cctx_load_dictionary(cctx: *mut c_void, dict: *const c_void, size: usize) -> usize {
    unsafe { ZSTD_CCtx_loadDictionary(cctx, dict, size) }
}

#[cfg(test)]
unsafe fn zstd_cctx_load_dictionary(
    _cctx: *mut c_void,
    _dict: *const c_void,
    _size: usize,
) -> usize {
    0
}

#[cfg(not(test))]
unsafe fn zstd_compress2(
    cctx: *mut c_void,
    dst: *mut c_void,
    dst_capacity: usize,
    src: *const c_void,
    src_size: usize,
) -> usize {
    unsafe { ZSTD_compress2(cctx, dst, dst_capacity, src, src_size) }
}

#[cfg(test)]
unsafe fn zstd_compress2(
    _cctx: *mut c_void,
    _dst: *mut c_void,
    _dst_capacity: usize,
    _src: *const c_void,
    _src_size: usize,
) -> usize {
    error_value(ZSTD_ERROR_GENERIC)
}

#[cfg(not(test))]
unsafe fn zstd_sizeof_cctx(cctx: *const c_void) -> usize {
    unsafe { ZSTD_sizeof_CCtx(cctx) }
}

#[cfg(test)]
unsafe fn zstd_sizeof_cctx(_cctx: *const c_void) -> usize {
    0
}

#[cfg(not(test))]
unsafe fn zstd_dctx_reset(dctx: *mut c_void) -> usize {
    unsafe { ZSTD_DCtx_reset(dctx, ZSTD_RESET_SESSION_AND_PARAMETERS) }
}

#[cfg(test)]
unsafe fn zstd_dctx_reset(_dctx: *mut c_void) -> usize {
    0
}

#[cfg(not(test))]
unsafe fn zstd_dctx_load_dictionary(dctx: *mut c_void, dict: *const c_void, size: usize) -> usize {
    unsafe { ZSTD_DCtx_loadDictionary(dctx, dict, size) }
}

#[cfg(test)]
unsafe fn zstd_dctx_load_dictionary(
    _dctx: *mut c_void,
    _dict: *const c_void,
    _size: usize,
) -> usize {
    0
}

#[cfg(not(test))]
unsafe fn zstd_decompress_stream(
    dctx: *mut c_void,
    output: *mut ZSTD_outBuffer,
    input: *mut ZSTD_inBuffer,
) -> usize {
    unsafe { ZSTD_decompressStream(dctx, output, input) }
}

#[cfg(test)]
unsafe fn zstd_decompress_stream(
    _dctx: *mut c_void,
    _output: *mut ZSTD_outBuffer,
    _input: *mut ZSTD_inBuffer,
) -> usize {
    error_value(ZSTD_ERROR_GENERIC)
}

#[cfg(not(test))]
unsafe fn zstd_version() -> String {
    let ptr = unsafe { ZSTD_versionString() };
    if ptr.is_null() {
        String::new()
    } else {
        unsafe { CStr::from_ptr(ptr) }
            .to_string_lossy()
            .into_owned()
    }
}

#[cfg(test)]
unsafe fn zstd_version() -> String {
    "test".to_owned()
}

#[cfg(not(test))]
unsafe fn timed_state_create(total_ms: c_uint) -> *mut BMK_timedFnState_t {
    unsafe { BMK_createTimedFnState(total_ms, BMK_RUNTEST_DEFAULT_MS) }
}

#[cfg(test)]
unsafe fn timed_state_create(_total_ms: c_uint) -> *mut BMK_timedFnState_t {
    ptr::null_mut()
}

#[cfg(not(test))]
unsafe fn timed_state_free(state: *mut BMK_timedFnState_t) {
    unsafe { BMK_freeTimedFnState(state) };
}

#[cfg(test)]
unsafe fn timed_state_free(_state: *mut BMK_timedFnState_t) {}

#[cfg(not(test))]
unsafe fn timed_state_run(
    state: *mut BMK_timedFnState_t,
    params: BMK_benchParams_t,
) -> BMK_runOutcome_t {
    unsafe { BMK_benchTimedFn(state, params) }
}

#[cfg(test)]
unsafe fn timed_state_run(
    _state: *mut BMK_timedFnState_t,
    _params: BMK_benchParams_t,
) -> BMK_runOutcome_t {
    BMK_runOutcome_t {
        internal_never_ever_use_directly: BMK_runTime_t::default(),
        error_result_never_ever_use_directly: error_value(ZSTD_ERROR_GENERIC),
        error_tag_never_ever_use_directly: 1,
    }
}

#[cfg(not(test))]
unsafe fn timed_state_complete(state: *const BMK_timedFnState_t) -> bool {
    unsafe { BMK_isCompleted_TimedFn(state) != 0 }
}

#[cfg(test)]
unsafe fn timed_state_complete(_state: *const BMK_timedFnState_t) -> bool {
    true
}

#[cfg(not(test))]
unsafe fn generate_rdg(buffer: *mut c_void, size: usize, compressibility: f64) {
    unsafe { RDG_genBuffer(buffer, size, compressibility, 0.0, 0) };
}

#[cfg(test)]
unsafe fn generate_rdg(buffer: *mut c_void, size: usize, _compressibility: f64) {
    if size != 0 {
        unsafe { ptr::write_bytes(buffer.cast::<u8>(), b'0', size) };
    }
}

#[cfg(not(test))]
unsafe fn generate_lorem(buffer: *mut c_void, size: usize) {
    unsafe { LOREM_genBuffer(buffer, size, 0) };
}

#[cfg(test)]
unsafe fn generate_lorem(buffer: *mut c_void, size: usize) {
    if size != 0 {
        unsafe { ptr::write_bytes(buffer.cast::<u8>(), b'l', size) };
    }
}

#[inline]
fn run_succeeded(outcome: BMK_runOutcome_t) -> bool {
    outcome.error_tag_never_ever_use_directly == 0
}

#[inline]
fn c_speed(src_size: usize, nano_sec_per_run: f64) -> u64 {
    if nano_sec_per_run <= 0.0 || !nano_sec_per_run.is_finite() {
        return 0;
    }
    ((src_size as f64 * 1_000_000_000.0) / nano_sec_per_run) as u64
}

fn display_label(name: &str) -> String {
    let name = display_name(name);
    let bytes = name.as_bytes();
    if bytes.len() <= 17 {
        return name.to_owned();
    }
    String::from_utf8_lossy(&bytes[bytes.len() - 17..]).into_owned()
}

fn ratio_digits(ratio: f64) -> usize {
    1 + usize::from(ratio < 100.0) + usize::from(ratio < 10.0)
}

fn output_progress(
    display_level: c_int,
    mark: &str,
    name: &str,
    src_size: usize,
    c_size: usize,
    ratio: f64,
    c_speed: u64,
    d_speed: Option<u64>,
) {
    if display_level < 2 {
        return;
    }
    let label = display_label(name);
    let ratio = format!("{:.*}", ratio_digits(ratio), ratio);
    let c_speed_digits = usize::from(c_speed < 10 * MB_UNIT);
    let c_speed = format!("{:.*}", c_speed_digits + 1, c_speed as f64 / MB_UNIT as f64);
    let mut stdout = io::stdout();
    if let Some(d_speed) = d_speed {
        let d_speed = format!("{d_speed:6.1}", d_speed = d_speed as f64 / MB_UNIT as f64);
        let _ = write!(
            stdout,
            "{mark:>2}-{label:<17} :{src_size:>10} ->{c_size:>10} (x{ratio:>5}), {c_speed:>6} MB/s, {d_speed} MB/s\r"
        );
    } else {
        let _ = write!(
            stdout,
            "{mark:>2}-{label:<17} :{src_size:>10} ->{c_size:>10} (x{ratio:>5}), {c_speed:>6} MB/s\r"
        );
    }
    let _ = stdout.flush();
}

fn set_realtime_priority() {
    #[cfg(unix)]
    unsafe {
        let _ = libc::setpriority(libc::PRIO_PROCESS, 0, -20);
    }
}

#[inline]
fn try_bytes(size: usize) -> Result<Vec<u8>, ()> {
    let mut result = Vec::new();
    result.try_reserve_exact(size).map_err(|_| ())?;
    result.resize(size, 0);
    Ok(result)
}

#[inline]
fn try_vec<T>(capacity: usize) -> Result<Vec<T>, ()> {
    let mut result = Vec::new();
    result.try_reserve_exact(capacity).map_err(|_| ())?;
    Ok(result)
}

unsafe fn c_string_path(value: *const c_char) -> Option<PathBuf> {
    if value.is_null() {
        return None;
    }
    let bytes = unsafe { CStr::from_ptr(value) }.to_bytes();
    #[cfg(unix)]
    {
        Some(PathBuf::from(std::ffi::OsStr::from_bytes(bytes)))
    }
    #[cfg(not(unix))]
    {
        Some(PathBuf::from(String::from_utf8_lossy(bytes).into_owned()))
    }
}

fn display_name(value: &str) -> &str {
    value.rsplit(['/', '\\']).next().unwrap_or(value)
}

fn zstd_error_callback(result: usize) -> c_uint {
    c_uint::from(is_zstd_error(result))
}

unsafe extern "C" fn zstd_error_callback_abi(result: usize) -> c_uint {
    zstd_error_callback(result)
}

#[derive(Clone, Copy)]
struct BenchCodecContext {
    cctx: *mut c_void,
    dctx: *mut c_void,
    dict: *const c_void,
    dict_size: usize,
    c_level: c_int,
    cparams: ZSTD_compressionParameters,
    adv: BMK_advancedParams_t,
    init_error: usize,
}

unsafe fn set_c_parameter(context: &mut BenchCodecContext, param: c_int, value: c_int) -> bool {
    let result = unsafe { zstd_cctx_set_parameter(context.cctx, param, value) };
    if is_zstd_error(result) {
        context.init_error = result;
        false
    } else {
        true
    }
}

unsafe fn initialize_compression(context: &mut BenchCodecContext) -> usize {
    context.init_error = 0;
    let reset = unsafe { zstd_cctx_reset(context.cctx) };
    if is_zstd_error(reset) {
        context.init_error = reset;
        return reset;
    }

    let workers = if context.adv.nbWorkers == 1 {
        0
    } else {
        context.adv.nbWorkers
    };
    let parameters = [
        (ZSTD_C_NB_WORKERS, workers),
        (ZSTD_C_COMPRESSION_LEVEL, context.c_level),
        (ZSTD_C_USE_ROW_MATCH_FINDER, context.adv.useRowMatchFinder),
        (ZSTD_C_ENABLE_LDM, context.adv.ldmFlag),
        (ZSTD_C_LDM_MIN_MATCH, context.adv.ldmMinMatch),
        (ZSTD_C_LDM_HASH_LOG, context.adv.ldmHashLog),
        (ZSTD_C_LDM_BUCKET_SIZE_LOG, context.adv.ldmBucketSizeLog),
        (ZSTD_C_LDM_HASH_RATE_LOG, context.adv.ldmHashRateLog),
        (ZSTD_C_WINDOW_LOG, context.cparams.windowLog as c_int),
        (ZSTD_C_HASH_LOG, context.cparams.hashLog as c_int),
        (ZSTD_C_CHAIN_LOG, context.cparams.chainLog as c_int),
        (ZSTD_C_SEARCH_LOG, context.cparams.searchLog as c_int),
        (ZSTD_C_MIN_MATCH, context.cparams.minMatch as c_int),
        (ZSTD_C_TARGET_LENGTH, context.cparams.targetLength as c_int),
        (
            ZSTD_C_LITERAL_COMPRESSION_MODE,
            context.adv.literalCompressionMode,
        ),
        (ZSTD_C_STRATEGY, context.cparams.strategy),
        (
            ZSTD_C_TARGET_C_BLOCK_SIZE,
            context.adv.targetCBlockSize as c_int,
        ),
    ];
    for (param, value) in parameters {
        if !unsafe { set_c_parameter(context, param, value) } {
            return context.init_error;
        }
    }

    let result =
        unsafe { zstd_cctx_load_dictionary(context.cctx, context.dict, context.dict_size) };
    if is_zstd_error(result) {
        context.init_error = result;
    }
    result
}

unsafe fn initialize_decompression(context: &mut BenchCodecContext) -> usize {
    context.init_error = 0;
    let reset = unsafe { zstd_dctx_reset(context.dctx) };
    if is_zstd_error(reset) {
        context.init_error = reset;
        return reset;
    }
    let result =
        unsafe { zstd_dctx_load_dictionary(context.dctx, context.dict, context.dict_size) };
    if is_zstd_error(result) {
        context.init_error = result;
    }
    result
}

unsafe extern "C" fn initialize_compression_abi(payload: *mut c_void) -> usize {
    let context = unsafe { &mut *payload.cast::<BenchCodecContext>() };
    unsafe { initialize_compression(context) }
}

unsafe extern "C" fn initialize_decompression_abi(payload: *mut c_void) -> usize {
    let context = unsafe { &mut *payload.cast::<BenchCodecContext>() };
    unsafe { initialize_decompression(context) }
}

unsafe extern "C" fn compress_block_abi(
    src: *const c_void,
    src_size: usize,
    dst: *mut c_void,
    dst_capacity: usize,
    payload: *mut c_void,
) -> usize {
    let context = unsafe { &mut *payload.cast::<BenchCodecContext>() };
    if context.init_error != 0 {
        return context.init_error;
    }
    unsafe { zstd_compress2(context.cctx, dst, dst_capacity, src, src_size) }
}

unsafe extern "C" fn decompress_block_abi(
    src: *const c_void,
    src_size: usize,
    dst: *mut c_void,
    dst_capacity: usize,
    payload: *mut c_void,
) -> usize {
    let context = unsafe { &mut *payload.cast::<BenchCodecContext>() };
    if context.init_error != 0 {
        return context.init_error;
    }

    let mut input = ZSTD_inBuffer {
        src,
        size: src_size,
        pos: 0,
    };
    let mut output = ZSTD_outBuffer {
        dst,
        size: dst_capacity,
        pos: 0,
    };
    let mut remaining = 1usize;
    while remaining != 0 {
        if output.pos == output.size {
            return error_value(ZSTD_ERROR_DST_SIZE_TOO_SMALL);
        }
        remaining = unsafe { zstd_decompress_stream(context.dctx, &mut output, &mut input) };
        if is_zstd_error(remaining) {
            return remaining;
        }
    }
    output.pos
}

struct BlockSpec {
    src_offset: usize,
    src_size: usize,
    result_size: usize,
}

struct BenchBuffers {
    src_ptrs: Vec<*const c_void>,
    src_sizes: Vec<usize>,
    c_ptrs: Vec<*mut c_void>,
    c_capacities: Vec<usize>,
    c_sizes: Vec<usize>,
    result_ptrs: Vec<*mut c_void>,
    result_capacities: Vec<usize>,
    _compressed_storage: Option<Vec<u8>>,
    result_storage: Vec<u8>,
}

unsafe fn build_block_specs(
    src: *const c_void,
    src_size: usize,
    file_sizes: &[usize],
    mode: BMK_mode_t,
    block_size: usize,
) -> Result<(Vec<BlockSpec>, usize), c_int> {
    let mut specs = try_vec(file_sizes.len()).map_err(|_| 31)?;
    let mut source_offset = 0usize;
    let mut logical_size = src_size;

    if mode == BMK_decodeOnly {
        logical_size = 0;
        for &file_size in file_sizes {
            if source_offset.checked_add(file_size).is_none() {
                return Err(32);
            }
            let file_src = unsafe { (src.cast::<u8>()).add(source_offset) }.cast();
            let decoded = unsafe { zstd_find_decompressed_size(file_src, file_size) };
            if decoded == ZSTD_CONTENTSIZE_UNKNOWN || decoded == ZSTD_CONTENTSIZE_ERROR {
                return Err(32);
            }
            let decoded = usize::try_from(decoded).map_err(|_| 32)?;
            logical_size = logical_size.checked_add(decoded).ok_or(32)?;
            specs.push(BlockSpec {
                src_offset: source_offset,
                src_size: file_size,
                result_size: decoded,
            });
            source_offset += file_size;
        }
        return Ok((specs, logical_size));
    }

    for &file_size in file_sizes {
        let mut remaining = file_size;
        while remaining != 0 {
            let this_size = remaining.min(block_size);
            specs.push(BlockSpec {
                src_offset: source_offset,
                src_size: this_size,
                result_size: this_size,
            });
            source_offset += this_size;
            remaining -= this_size;
        }
    }
    Ok((specs, logical_size))
}

unsafe fn build_bench_buffers(
    src: *const c_void,
    src_size: usize,
    dst: *mut c_void,
    dst_capacity: usize,
    specs: &[BlockSpec],
    mode: BMK_mode_t,
) -> Result<BenchBuffers, c_int> {
    let mut compressed_size = 0usize;
    let mut compressed_capacities = try_vec(specs.len()).map_err(|_| 31)?;
    for spec in specs {
        let capacity = if mode == BMK_decodeOnly {
            spec.src_size
        } else {
            let bound = unsafe { zstd_compress_bound(spec.src_size) };
            if is_zstd_error(bound) {
                return Err(31);
            }
            bound
        };
        compressed_size = compressed_size.checked_add(capacity).ok_or(31)?;
        compressed_capacities.push(capacity);
    }

    if (dst.is_null()) != (dst_capacity == 0) {
        return Err(32);
    }
    if !dst.is_null() && dst_capacity < compressed_size {
        return Err(32);
    }

    let compressed_storage = if dst.is_null() {
        let mut capacity = compressed_size;
        if mode != BMK_decodeOnly {
            capacity = capacity
                .checked_add(specs.len().saturating_mul(1024))
                .ok_or(31)?;
        }
        Some(try_bytes(capacity).map_err(|_| 31)?)
    } else {
        None
    };
    let compressed_base = if let Some(storage) = compressed_storage.as_ref() {
        storage.as_ptr().cast_mut()
    } else {
        dst.cast::<u8>()
    };

    if mode == BMK_decodeOnly && src_size != 0 {
        unsafe { ptr::copy_nonoverlapping(src.cast::<u8>(), compressed_base, src_size) };
    }

    let mut result_size = 0usize;
    for spec in specs {
        result_size = result_size.checked_add(spec.result_size).ok_or(31)?;
    }
    let result_storage = try_bytes(result_size).map_err(|_| 31)?;
    let result_base = result_storage.as_ptr().cast_mut();

    let mut src_ptrs = try_vec(specs.len()).map_err(|_| 31)?;
    let mut src_sizes = try_vec(specs.len()).map_err(|_| 31)?;
    let mut c_ptrs = try_vec(specs.len()).map_err(|_| 31)?;
    let mut c_capacities = try_vec(specs.len()).map_err(|_| 31)?;
    let mut c_sizes = try_vec(specs.len()).map_err(|_| 31)?;
    let mut result_ptrs = try_vec(specs.len()).map_err(|_| 31)?;
    let mut result_capacities = try_vec(specs.len()).map_err(|_| 31)?;
    let mut c_offset = 0usize;
    let mut result_offset = 0usize;

    for (index, spec) in specs.iter().enumerate() {
        let src_ptr = unsafe { src.cast::<u8>().add(spec.src_offset) }.cast();
        let c_ptr = unsafe { compressed_base.add(c_offset) }.cast();
        let result_ptr = unsafe { result_base.add(result_offset) }.cast();
        src_ptrs.push(src_ptr);
        src_sizes.push(spec.src_size);
        c_ptrs.push(c_ptr);
        c_capacities.push(compressed_capacities[index]);
        c_sizes.push(if mode == BMK_decodeOnly {
            spec.src_size
        } else {
            0
        });
        result_ptrs.push(result_ptr);
        result_capacities.push(spec.result_size);
        c_offset += compressed_capacities[index];
        result_offset += spec.result_size;
    }

    Ok(BenchBuffers {
        src_ptrs,
        src_sizes,
        c_ptrs,
        c_capacities,
        c_sizes,
        result_ptrs,
        result_capacities,
        _compressed_storage: compressed_storage,
        result_storage,
    })
}

unsafe fn bench_mem_advanced(
    src: *const c_void,
    src_size: usize,
    dst: *mut c_void,
    dst_capacity: usize,
    file_sizes: &[usize],
    c_level: c_int,
    cparams: ZSTD_compressionParameters,
    dict: *const c_void,
    dict_size: usize,
    display_level: c_int,
    name: &str,
    adv: BMK_advancedParams_t,
) -> BMK_benchOutcome_t {
    if (dst.is_null()) != (dst_capacity == 0) {
        return bench_error(32);
    }
    if file_sizes.is_empty() || src.is_null() && src_size != 0 {
        return bench_error(32);
    }
    let file_total = file_sizes
        .iter()
        .try_fold(0usize, |total, size| total.checked_add(*size));
    if file_total != Some(src_size) {
        return bench_error(32);
    }
    if dict_size != 0 && dict.is_null() {
        return bench_error(32);
    }
    if src_size == 0 {
        return bench_error(31);
    }

    let block_size = if adv.mode != BMK_decodeOnly && adv.blockSize >= 32 {
        adv.blockSize
    } else {
        src_size
    };
    let (specs, logical_size) = match unsafe {
        build_block_specs(src, src_size, file_sizes, adv.mode, block_size.max(1))
    } {
        Ok(value) => value,
        Err(code) => return bench_error(code),
    };
    if specs.is_empty() {
        return bench_error(31);
    }
    let mut buffers =
        match unsafe { build_bench_buffers(src, src_size, dst, dst_capacity, &specs, adv.mode) } {
            Ok(value) => value,
            Err(code) => return bench_error(code),
        };

    let cctx = unsafe { zstd_create_cctx() };
    let dctx = unsafe { zstd_create_dctx() };
    if cctx.is_null() || dctx.is_null() {
        unsafe {
            zstd_free_cctx(cctx);
            zstd_free_dctx(dctx);
        }
        return bench_error(31);
    }

    let total_ms = adv.nbSeconds.wrapping_mul(1_000);
    let compression_state = unsafe { timed_state_create(total_ms) };
    let decompression_state = unsafe { timed_state_create(total_ms) };
    if compression_state.is_null() || decompression_state.is_null() {
        unsafe {
            timed_state_free(compression_state);
            timed_state_free(decompression_state);
            zstd_free_cctx(cctx);
            zstd_free_dctx(dctx);
        }
        return bench_error(31);
    }

    let mut context = BenchCodecContext {
        cctx,
        dctx,
        dict,
        dict_size,
        c_level,
        cparams,
        adv,
        init_error: 0,
    };
    let payload = (&mut context as *mut BenchCodecContext).cast::<c_void>();
    let mut compression_params = BMK_benchParams_t {
        benchFn: Some(compress_block_abi),
        benchPayload: payload,
        initFn: Some(initialize_compression_abi),
        initPayload: payload,
        errorFn: Some(zstd_error_callback_abi),
        blockCount: buffers.src_ptrs.len(),
        srcBuffers: buffers.src_ptrs.as_ptr(),
        srcSizes: buffers.src_sizes.as_ptr(),
        dstBuffers: buffers.c_ptrs.as_ptr(),
        dstCapacities: buffers.c_capacities.as_ptr(),
        blockResults: buffers.c_sizes.as_mut_ptr(),
    };
    let decompression_params = BMK_benchParams_t {
        benchFn: Some(decompress_block_abi),
        benchPayload: payload,
        initFn: Some(initialize_decompression_abi),
        initPayload: payload,
        errorFn: Some(zstd_error_callback_abi),
        blockCount: buffers.c_ptrs.len(),
        srcBuffers: buffers.c_ptrs.as_ptr().cast(),
        srcSizes: buffers.c_sizes.as_ptr(),
        dstBuffers: buffers.result_ptrs.as_ptr(),
        dstCapacities: buffers.result_capacities.as_ptr(),
        blockResults: ptr::null_mut(),
    };

    let mut compression_done = adv.mode == BMK_decodeOnly;
    let mut decompression_done = adv.mode == BMK_compressOnly;
    let mut result = BMK_benchResult_t::default();
    let mut c_size = if adv.mode == BMK_decodeOnly {
        src_size
    } else {
        0
    };
    let mut ratio = if c_size == 0 {
        0.0
    } else {
        logical_size as f64 / c_size as f64
    };

    let marks = [" |", " /", " =", " \\"];
    let mut mark_number = 0usize;
    if display_level >= 2 {
        let mut stdout = io::stdout();
        let _ = write!(stdout, "\r{:>70}\r", "");
        let _ = stdout.flush();
        output_progress(
            display_level,
            marks[mark_number],
            name,
            logical_size,
            c_size,
            ratio,
            result.cSpeed,
            None,
        );
    }

    while !(compression_done && decompression_done) {
        if !compression_done {
            let outcome = unsafe { timed_state_run(compression_state, compression_params) };
            if !run_succeeded(outcome) {
                unsafe {
                    timed_state_free(compression_state);
                    timed_state_free(decompression_state);
                    zstd_free_cctx(cctx);
                    zstd_free_dctx(dctx);
                }
                return bench_error(30);
            }
            let runtime = outcome.internal_never_ever_use_directly;
            c_size = runtime.sumOfReturn;
            ratio = if c_size == 0 {
                0.0
            } else {
                logical_size as f64 / c_size as f64
            };
            result.cSize = c_size;
            result.cSpeed = result
                .cSpeed
                .max(c_speed(logical_size, runtime.nanoSecPerRun));
            output_progress(
                display_level,
                marks[mark_number],
                name,
                logical_size,
                c_size,
                ratio,
                result.cSpeed,
                None,
            );
            compression_done = unsafe { timed_state_complete(compression_state) };
        }

        if !decompression_done {
            let outcome = unsafe { timed_state_run(decompression_state, decompression_params) };
            if !run_succeeded(outcome) {
                unsafe {
                    timed_state_free(compression_state);
                    timed_state_free(decompression_state);
                    zstd_free_cctx(cctx);
                    zstd_free_dctx(dctx);
                }
                return bench_error(30);
            }
            let runtime = outcome.internal_never_ever_use_directly;
            result.dSpeed = result
                .dSpeed
                .max(c_speed(logical_size, runtime.nanoSecPerRun));
            output_progress(
                display_level,
                marks[mark_number],
                name,
                logical_size,
                c_size,
                ratio,
                result.cSpeed,
                Some(result.dSpeed),
            );
            decompression_done = unsafe { timed_state_complete(decompression_state) };
        }
        mark_number = (mark_number + 1) % marks.len();
    }

    if adv.mode == BMK_both {
        let result_bytes = &buffers.result_storage;
        let source_bytes = unsafe { std::slice::from_raw_parts(src.cast::<u8>(), src_size) };
        if result_bytes != source_bytes && display_level >= 1 {
            eprintln!(
                "!!! WARNING !!! {:>17} : Invalid Checksum",
                display_name(name)
            );
        }
    }

    if display_level == 1 {
        let c_speed = result.cSpeed as f64 / MB_UNIT as f64;
        let d_speed = result.dSpeed as f64 / MB_UNIT as f64;
        if adv.additionalParam != 0 {
            println!(
                "-{c_level:<3}{c_size:>11} ({ratio:5.3}) {c_speed:6.2} MB/s {d_speed:6.1} MB/s  {} (param={})",
                display_name(name), adv.additionalParam
            );
        } else {
            println!(
                "-{c_level:<3}{c_size:>11} ({ratio:5.3}) {c_speed:6.2} MB/s {d_speed:6.1} MB/s  {}",
                display_name(name)
            );
        }
    } else if display_level >= 2 {
        println!("{c_level:2}#");
    }

    let window_memory = if cparams.windowLog < usize::BITS {
        1usize << cparams.windowLog
    } else {
        0
    };
    result.cMem = window_memory.saturating_add(unsafe { zstd_sizeof_cctx(cctx) });

    unsafe {
        timed_state_free(compression_state);
        timed_state_free(decompression_state);
        zstd_free_cctx(cctx);
        zstd_free_dctx(dctx);
    }
    /* Keep buffers alive until every callback and context has finished. */
    let _ = (&mut buffers, &mut compression_params, ratio);
    result_outcome(result)
}

fn bench_error(_tag: c_int) -> BMK_benchOutcome_t {
    BMK_benchOutcome_t {
        internal_never_ever_use_directly: BMK_benchResult_t::default(),
        tag: 1,
    }
}

fn result_outcome(result: BMK_benchResult_t) -> BMK_benchOutcome_t {
    BMK_benchOutcome_t {
        internal_never_ever_use_directly: result,
        tag: 0,
    }
}

unsafe fn bench_c_levels(
    src: *const c_void,
    src_size: usize,
    file_sizes: &[usize],
    start_level: c_int,
    end_level: c_int,
    cparams: ZSTD_compressionParameters,
    dict: *const c_void,
    dict_size: usize,
    display_level: c_int,
    name: &str,
    adv: BMK_advancedParams_t,
) -> c_int {
    if end_level > unsafe { zstd_max_c_level() } || end_level < start_level {
        if display_level >= 1 {
            eprintln!(
                "{}",
                if end_level < start_level {
                    "Invalid Compression Level Range"
                } else {
                    "Invalid Compression Level"
                }
            );
        }
        return 15;
    }
    if adv.realTime != 0 {
        if display_level >= 2 {
            eprintln!("Note : switching to real-time priority");
        }
        set_realtime_priority();
    }
    if display_level == 1 && adv.additionalParam == 0 {
        let version = unsafe { zstd_version() };
        println!(
            "bench {version} : input {} bytes, {} seconds, {} KB blocks",
            src_size,
            adv.nbSeconds,
            adv.blockSize >> 10
        );
    }

    let mut level = start_level;
    loop {
        let outcome = unsafe {
            bench_mem_advanced(
                src,
                src_size,
                ptr::null_mut(),
                0,
                file_sizes,
                level,
                cparams,
                dict,
                dict_size,
                display_level,
                name,
                adv,
            )
        };
        if BMK_isSuccessful_benchOutcome(outcome) == 0 {
            return 1;
        }
        if level == end_level {
            break;
        }
        level = level.saturating_add(1);
    }
    0
}

fn max_memory() -> u64 {
    if usize::BITS == 32 {
        (2u64 << 30) - (64u64 << 20)
    } else {
        1u64 << (usize::BITS - 31)
    }
}

fn find_max_memory(required_memory: u64) -> usize {
    let step = 64u64 << 20;
    let mut required = ((required_memory >> 26) + 1) << 26;
    required = required.saturating_add(step).min(max_memory());

    loop {
        let allocation = usize::try_from(required)
            .ok()
            .map(|size| unsafe { libc::malloc(size) });
        required = required.saturating_sub(step);
        if let Some(allocation) = allocation {
            if !allocation.is_null() {
                unsafe { libc::free(allocation) };
                return required as usize;
            }
        }
        if required == 0 {
            return 0;
        }
    }
}

fn load_files(
    buffer: &mut [u8],
    file_sizes: &mut [usize],
    paths: &[PathBuf],
    display_level: c_int,
) -> Result<(), c_int> {
    let mut position = 0usize;
    let mut total_size = 0usize;

    for (index, path) in paths.iter().enumerate() {
        let metadata = match fs::metadata(path) {
            Ok(metadata) => metadata,
            Err(_) => return Err(1),
        };
        if metadata.is_dir() {
            if display_level >= 2 {
                println!("Ignoring {} directory...       ", path.display());
            }
            file_sizes[index] = 0;
            continue;
        }
        if !metadata.is_file() {
            file_sizes[index] = 0;
            continue;
        }

        let mut file_size = usize::try_from(metadata.len()).map_err(|_| 1)?;
        let available = buffer.len().saturating_sub(position);
        let truncated = file_size > available;
        if truncated {
            file_size = available;
        }

        if display_level >= 2 {
            print!("Loading {}...       \r", path.display());
            let _ = io::stdout().flush();
        }
        let mut file = fs::File::open(path).map_err(|_| 1)?;
        file.read_exact(&mut buffer[position..position + file_size])
            .map_err(|_| 1)?;
        file_sizes[index] = file_size;
        position += file_size;
        total_size += file_size;

        if truncated {
            break;
        }
    }

    if total_size == 0 {
        return Err(12);
    }
    Ok(())
}

unsafe fn collect_file_inputs(
    file_names: *const *const c_char,
    nb_files: c_uint,
    display_level: c_int,
) -> Result<(Vec<u8>, Vec<usize>, String), c_int> {
    if file_names.is_null() || nb_files == 0 {
        return Err(13);
    }
    let mut paths = try_vec(nb_files as usize).map_err(|_| 16)?;
    for index in 0..nb_files as usize {
        let path = unsafe { c_string_path(*file_names.add(index)) }.ok_or(15)?;
        paths.push(path);
    }

    let mut total_size = 0u64;
    let mut sizes = try_vec(paths.len()).map_err(|_| 16)?;
    for path in &paths {
        let metadata = fs::metadata(path).map_err(|_| 15)?;
        if !metadata.is_file() {
            return Err(15);
        }
        total_size = total_size.checked_add(metadata.len()).ok_or(15)?;
        sizes.push(usize::try_from(metadata.len()).map_err(|_| 15)?);
    }
    if total_size == 0 {
        return Err(1);
    }
    let benched_size = (find_max_memory(total_size.saturating_mul(3)) / 3)
        .min(usize::try_from(total_size).map_err(|_| 20)?);
    if benched_size == 0 {
        return Err(20);
    }
    if u64::try_from(benched_size).unwrap_or(u64::MAX) < total_size {
        eprintln!(
            "Not enough memory; testing {} MB only...",
            benched_size >> 20
        );
    }
    let mut source = try_bytes(benched_size).map_err(|_| 20)?;
    sizes.fill(0);
    match load_files(&mut source, &mut sizes, &paths, display_level) {
        Ok(()) => {}
        Err(12) => return Err(1),
        Err(_) => return Err(1),
    }
    let name = if nb_files > 1 {
        format!(" {nb_files} files")
    } else {
        unsafe { CStr::from_ptr(*file_names) }
            .to_string_lossy()
            .into_owned()
    };
    Ok((source, sizes, name))
}

unsafe fn collect_dictionary(
    dict_file_name: *const c_char,
    display_level: c_int,
) -> Result<Vec<u8>, c_int> {
    let Some(path) = (unsafe { c_string_path(dict_file_name) }) else {
        return Ok(Vec::new());
    };
    let metadata = fs::metadata(&path).map_err(|_| 17)?;
    if !metadata.is_file() {
        return Err(17);
    }
    let size = usize::try_from(metadata.len()).map_err(|_| 18)?;
    if size > 64 * 1024 * 1024 {
        return Err(18);
    }
    let mut data = try_bytes(size).map_err(|_| 19)?;
    let mut file_sizes = [0usize; 1];
    match load_files(
        data.as_mut_slice(),
        &mut file_sizes,
        std::slice::from_ref(&path),
        display_level,
    ) {
        Ok(()) => {}
        Err(_) => return Err(1),
    }
    Ok(data)
}

/// Returns the defaults used by the non-advanced benchmark entry points.
#[no_mangle]
pub extern "C" fn BMK_initAdvancedParams() -> BMK_advancedParams_t {
    BMK_advancedParams_t::default()
}

/// Returns nonzero when a benchmark outcome contains a valid result.
#[no_mangle]
pub extern "C" fn BMK_isSuccessful_benchOutcome(outcome: BMK_benchOutcome_t) -> c_int {
    c_int::from(outcome.tag == 0)
}

/// Extracts a valid result and aborts when called on an error outcome.
#[no_mangle]
pub extern "C" fn BMK_extract_benchResult(outcome: BMK_benchOutcome_t) -> BMK_benchResult_t {
    if outcome.tag != 0 {
        std::process::abort();
    }
    outcome.internal_never_ever_use_directly
}

/// Core memory-to-memory benchmark entry point from `benchzstd.h`.
#[no_mangle]
pub unsafe extern "C" fn BMK_benchMemAdvanced(
    srcBuffer: *const c_void,
    srcSize: usize,
    dstBuffer: *mut c_void,
    dstCapacity: usize,
    fileSizes: *const usize,
    nbFiles: c_uint,
    cLevel: c_int,
    comprParams: *const ZSTD_compressionParameters,
    dictBuffer: *const c_void,
    dictBufferSize: usize,
    displayLevel: c_int,
    displayName: *const c_char,
    adv: *const BMK_advancedParams_t,
) -> BMK_benchOutcome_t {
    if fileSizes.is_null() || comprParams.is_null() || adv.is_null() {
        return bench_error(32);
    }
    let file_sizes = unsafe { std::slice::from_raw_parts(fileSizes, nbFiles as usize) };
    let name = if displayName.is_null() {
        String::new()
    } else {
        unsafe { CStr::from_ptr(displayName) }
            .to_string_lossy()
            .into_owned()
    };
    unsafe {
        bench_mem_advanced(
            srcBuffer,
            srcSize,
            dstBuffer,
            dstCapacity,
            file_sizes,
            cLevel,
            *comprParams,
            dictBuffer,
            dictBufferSize,
            displayLevel,
            &name,
            *adv,
        )
    }
}

/// Default-parameter memory-to-memory benchmark entry point.
#[no_mangle]
pub unsafe extern "C" fn BMK_benchMem(
    srcBuffer: *const c_void,
    srcSize: usize,
    fileSizes: *const usize,
    nbFiles: c_uint,
    cLevel: c_int,
    comprParams: *const ZSTD_compressionParameters,
    dictBuffer: *const c_void,
    dictBufferSize: usize,
    displayLevel: c_int,
    displayName: *const c_char,
) -> BMK_benchOutcome_t {
    let adv = BMK_initAdvancedParams();
    unsafe {
        BMK_benchMemAdvanced(
            srcBuffer,
            srcSize,
            ptr::null_mut(),
            0,
            fileSizes,
            nbFiles,
            cLevel,
            comprParams,
            dictBuffer,
            dictBufferSize,
            displayLevel,
            displayName,
            &adv,
        )
    }
}

/// Loads files, optionally loads a dictionary, and benchmarks every level in
/// the requested range.
#[no_mangle]
pub unsafe extern "C" fn BMK_benchFilesAdvanced(
    fileNamesTable: *const *const c_char,
    nbFiles: c_uint,
    dictFileName: *const c_char,
    startCLevel: c_int,
    endCLevel: c_int,
    compressionParams: *const ZSTD_compressionParameters,
    displayLevel: c_int,
    adv: *const BMK_advancedParams_t,
) -> c_int {
    if nbFiles == 0 {
        return 13;
    }
    if compressionParams.is_null() || adv.is_null() {
        return 15;
    }
    if endCLevel > unsafe { zstd_max_c_level() } {
        return 14;
    }
    let (source, file_sizes, name) =
        match unsafe { collect_file_inputs(fileNamesTable, nbFiles, displayLevel) } {
            Ok(value) => value,
            Err(code) => return code,
        };
    let dictionary = match unsafe { collect_dictionary(dictFileName, displayLevel) } {
        Ok(value) => value,
        Err(code) => return code,
    };
    let dict_ptr = if dictionary.is_empty() {
        ptr::null()
    } else {
        dictionary.as_ptr().cast()
    };
    unsafe {
        bench_c_levels(
            source.as_ptr().cast(),
            source.len(),
            &file_sizes,
            startCLevel,
            endCLevel,
            *compressionParams,
            dict_ptr,
            dictionary.len(),
            displayLevel,
            &name,
            *adv,
        )
    }
}

/// Single-level convenience wrapper used by older callers.
#[no_mangle]
pub unsafe extern "C" fn BMK_benchFiles(
    fileNamesTable: *const *const c_char,
    nbFiles: c_uint,
    dictFileName: *const c_char,
    cLevel: c_int,
    compressionParams: *const ZSTD_compressionParameters,
    displayLevel: c_int,
) -> c_int {
    let adv = BMK_initAdvancedParams();
    unsafe {
        BMK_benchFilesAdvanced(
            fileNamesTable,
            nbFiles,
            dictFileName,
            cLevel,
            cLevel,
            compressionParams,
            displayLevel,
            &adv,
        )
    }
}

/// Generates a deterministic synthetic sample and benchmarks the requested
/// level range.  Negative compressibility selects the lorem ipsum generator,
/// matching `programs/benchzstd.c`.
#[no_mangle]
pub unsafe extern "C" fn BMK_syntheticTest(
    compressibility: f64,
    startingCLevel: c_int,
    endCLevel: c_int,
    compressionParams: *const ZSTD_compressionParameters,
    displayLevel: c_int,
    adv: *const BMK_advancedParams_t,
) -> c_int {
    if compressionParams.is_null() || adv.is_null() {
        return 15;
    }
    let adv_value = unsafe { *adv };
    let size = if adv_value.blockSize == 0 {
        10_000_000
    } else {
        adv_value.blockSize
    };
    let mut source = match try_bytes(size) {
        Ok(value) => value,
        Err(()) => return 16,
    };
    let name = if compressibility < 0.0 {
        unsafe { generate_lorem(source.as_mut_ptr().cast(), source.len()) };
        "Lorem ipsum".to_owned()
    } else {
        unsafe { generate_rdg(source.as_mut_ptr().cast(), source.len(), compressibility) };
        format!("Synthetic {}%", (compressibility * 100.0) as u32)
    };
    let file_sizes = [source.len()];
    unsafe {
        bench_c_levels(
            source.as_ptr().cast(),
            source.len(),
            &file_sizes,
            startingCLevel,
            endCLevel,
            *compressionParams,
            ptr::null(),
            0,
            displayLevel,
            &name,
            adv_value,
        )
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn advanced_defaults_match_benchzstd_c() {
        let defaults = BMK_initAdvancedParams();
        assert_eq!(defaults.mode, BMK_both);
        assert_eq!(defaults.nbSeconds, BMK_TIMETEST_DEFAULT_S);
        assert_eq!(defaults.blockSize, 0);
        assert_eq!(defaults.targetCBlockSize, 0);
        assert_eq!(defaults.nbWorkers, 0);
        assert_eq!(defaults.realTime, 0);
        assert_eq!(defaults.additionalParam, 0);
        assert_eq!(defaults.ldmFlag, 0);
        assert_eq!(defaults.ldmMinMatch, 0);
        assert_eq!(defaults.ldmHashLog, 0);
        assert_eq!(defaults.ldmBucketSizeLog, 0);
        assert_eq!(defaults.ldmHashRateLog, 0);
        assert_eq!(defaults.literalCompressionMode, ZSTD_ps_auto);
        assert_eq!(defaults.useRowMatchFinder, 0);
    }

    #[test]
    fn outcome_accessors_extract_only_valid_results() {
        let result = BMK_benchResult_t {
            cSize: 123,
            cSpeed: 456,
            dSpeed: 789,
            cMem: 321,
        };
        let valid = result_outcome(result);
        assert_eq!(BMK_isSuccessful_benchOutcome(valid), 1);
        assert_eq!(BMK_extract_benchResult(valid), result);

        let invalid = bench_error(30);
        assert_eq!(BMK_isSuccessful_benchOutcome(invalid), 0);
        assert_eq!(invalid.tag, 1);
    }

    #[test]
    fn outcome_layout_matches_header_variant() {
        assert_eq!(size_of::<BMK_benchResult_t>(), size_of::<usize>() * 4);
        assert_eq!(
            offset_of!(BMK_benchOutcome_t, tag),
            size_of::<BMK_benchResult_t>()
        );
    }

    #[test]
    fn public_parameter_layout_matches_the_c_headers() {
        assert_eq!(
            offset_of!(ZSTD_compressionParameters, strategy),
            6 * size_of::<u32>()
        );
        assert_eq!(
            offset_of!(BMK_advancedParams_t, targetCBlockSize),
            2 * size_of::<c_int>() + size_of::<usize>()
        );
    }

    #[test]
    fn block_partitioning_preserves_file_boundaries() {
        let (specs, logical_size) = unsafe {
            build_block_specs(ptr::null(), 10, &[3, 7], BMK_both, 4).expect("valid block partition")
        };
        assert_eq!(logical_size, 10);
        assert_eq!(
            specs
                .iter()
                .map(|spec| (spec.src_offset, spec.src_size))
                .collect::<Vec<_>>(),
            vec![(0, 3), (3, 4), (7, 3)]
        );
    }

    #[test]
    fn progress_labels_match_c_basename_and_tail_rules() {
        assert_eq!(display_label("/tmp/input"), "input");
        assert_eq!(
            display_label("/tmp/01234567890123456789"),
            "34567890123456789"
        );
    }
}