ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
2b703ae2a18c61398417ac0ca82abe7b273b652d
zstd-rs/rust/src/zstdmt_compress.rs
T
ddidderr 2b703ae2a1 fix(rust): satisfy layout assertions under clippy 
Use usize::BITS for the byte offset that Clippy recognizes as a bit-width expression on both the static-CCtx and MT finish projections. Update the focused resource-failure test to provide the compression-job last-job argument required by the callback signature. This keeps the ABI checks architecture-independent while making all-target clippy compile the new tests.

Test Plan:

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings

- git diff --cached --check
2026-07-20 16:14:10 +02:00

8512 lines
277 KiB
Rust
Raw Blame History

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
//! Rust-owned resource pools used by the multithreaded compressor.
//!
//! The serial LDM state, job descriptor fields, worker callback, and streaming
//! state still use private C layouts. `zstdmt_compress.c` therefore keeps
//! those operations and projects only allocation/lifecycle pieces and pure
//! sizing policy into this module. The entry points below are narrow C ABIs:
//! buffers, `ZSTD_CCtx *` values, and job descriptors remain opaque to Rust,
//! while allocation, reuse, table-capacity orchestration, and sizing policy
//! are Rust-owned. C retains ownership of private descriptor fields and
//! platform synchronization.
use std::mem::{self, offset_of, size_of, MaybeUninit};
use std::os::raw::{c_int, c_uint, c_void};
use std::ptr;
use std::sync::Mutex;
use crate::bits::ZSTD_highbit32;
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::mem::MEM_writeLE32;
use crate::zstd_compress::{ZSTD_frameProgression, ZSTD_rust_invalidateRepCodes};
use crate::zstd_compress_frame::ZSTD_rust_writeFrameHeader;
const ZSTDMT_JOBLOG_MAX: c_uint = if mem::size_of::<usize>() == 4 { 29 } else { 30 };
const ZSTD_WINDOWLOG_MAX: c_uint = if mem::size_of::<usize>() == 4 { 30 } else { 31 };
#[cfg(test)]
const DEFAULT_ZSTDMT_JOBSIZE_MIN: usize = 512 << 10;
#[cfg(test)]
const DEFAULT_ZSTDMT_JOBSIZE_MAX: usize = if mem::size_of::<usize>() == 4 {
512 << 20
} else {
1024 << 20
};
const ZSTD_FAST: c_int = 1;
const ZSTD_DFAST: c_int = 2;
const ZSTD_GREEDY: c_int = 3;
const ZSTD_LAZY: c_int = 4;
const ZSTD_LAZY2: c_int = 5;
const ZSTD_BTLAZY2: c_int = 6;
const ZSTD_BTOPT: c_int = 7;
const ZSTD_BTULTRA: c_int = 8;
const ZSTD_BTULTRA2: c_int = 9;
const ZSTD_PS_ENABLE: c_int = 1;
const ZSTD_PS_DISABLE: c_int = 2;
const RSYNC_LENGTH: usize = 32;
const RSYNC_MIN_BLOCK_LOG: usize = 17;
const RSYNC_MIN_BLOCK_SIZE: usize = 1 << RSYNC_MIN_BLOCK_LOG;
const PRIME8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463;
const ROLL_HASH_CHAR_OFFSET: u64 = 10;
const _: () = assert!(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH);
type ZstdMtCompressFn =
unsafe extern "C" fn(*mut c_void, *mut c_void, usize, *const c_void, usize) -> usize;
pub type ZSTDMT_chunkProgressFn = unsafe extern "C" fn(*mut c_void, usize, usize);
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_chunkProcessResult {
pub error: usize,
pub lastBlockSize: usize,
}
/// Per-job parameter policy applied before a worker initializes its private
/// compression context. The full `ZSTD_CCtx_params` remains C-owned; only the
/// three scalar fields changed by the MT worker policy cross the ABI.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobParameters {
pub checksumFlag: c_int,
pub ldmEnable: c_int,
pub nbWorkers: c_uint,
}
const _: () = {
assert!(offset_of!(ZSTDMT_compressionJobParameters, checksumFlag) == 0);
assert!(offset_of!(ZSTDMT_compressionJobParameters, ldmEnable) == size_of::<c_int>());
assert!(offset_of!(ZSTDMT_compressionJobParameters, nbWorkers) == 2 * size_of::<c_int>());
assert!(
size_of::<ZSTDMT_compressionJobParameters>()
== 2 * size_of::<c_int>() + size_of::<c_uint>()
);
};
#[inline]
fn prepare_compression_job_parameters(
job_id: c_uint,
checksum_flag: c_int,
_ldm_enable: c_int,
_nb_workers: c_uint,
) -> ZSTDMT_compressionJobParameters {
ZSTDMT_compressionJobParameters {
/* The first job writes the frame checksum; later jobs have their
* checksum accounted for by the serial MT state. */
checksumFlag: if job_id == 0 { checksum_flag } else { 0 },
/* LDM sequences are generated by the serial state, not per worker. */
ldmEnable: ZSTD_PS_DISABLE,
/* Worker contexts are initialized as single-threaded contexts. */
nbWorkers: 0,
}
}
/// Apply the pure per-worker MT parameter policy while C retains the complete
/// compression-parameter structure and all codec initialization.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_prepareCompressionJobParameters(
jobID: c_uint,
checksumFlag: c_int,
ldmEnable: c_int,
nbWorkers: c_uint,
) -> ZSTDMT_compressionJobParameters {
prepare_compression_job_parameters(jobID, checksumFlag, ldmEnable, nbWorkers)
}
/// Scalar job state used by the Rust compression-job scheduler.
///
/// The job descriptor, pools, synchronization, and codec state remain
/// private to C. Rust uses the frame-position flags and a scalar header
/// projection to choose the non-first-job header stage.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobProjection {
pub firstJob: c_uint,
pub lastJob: c_uint,
pub frameHeaderState: *const ZSTDMT_compressionJobFrameHeaderState,
}
/// Scalar projection for the non-first MT job frame header. The destination,
/// stage, and repcodes point at the worker-owned C storage; all header
/// serialization and repcode invalidation happen in Rust.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobFrameHeaderState {
dst: *mut c_void,
dst_capacity: usize,
stage: *mut c_int,
no_dict_id_flag: c_int,
checksum_flag: c_int,
content_size_flag: c_int,
format: c_int,
window_log: c_uint,
pledged_src_size_plus_one: u64,
dict_id: c_uint,
rep_codes: *mut c_uint,
}
const _: () = {
assert!(offset_of!(ZSTDMT_compressionJobProjection, firstJob) == 0);
assert!(offset_of!(ZSTDMT_compressionJobProjection, lastJob) == size_of::<c_uint>());
assert!(
offset_of!(ZSTDMT_compressionJobProjection, frameHeaderState) == size_of::<[c_uint; 2]>()
);
assert!(
size_of::<ZSTDMT_compressionJobProjection>()
== size_of::<[c_uint; 2]>() + size_of::<usize>()
);
assert!(offset_of!(ZSTDMT_compressionJobFrameHeaderState, dst) == 0);
assert!(offset_of!(ZSTDMT_compressionJobFrameHeaderState, dst_capacity) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_compressionJobFrameHeaderState, stage) == 2 * size_of::<usize>());
assert!(
offset_of!(ZSTDMT_compressionJobFrameHeaderState, no_dict_id_flag)
== 3 * size_of::<usize>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFrameHeaderState, checksum_flag)
== 3 * size_of::<usize>() + size_of::<c_int>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFrameHeaderState, content_size_flag)
== 3 * size_of::<usize>() + 2 * size_of::<c_int>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFrameHeaderState, format)
== 3 * size_of::<usize>() + 3 * size_of::<c_int>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFrameHeaderState, window_log)
== 3 * size_of::<usize>() + 4 * size_of::<c_int>()
);
assert!(
offset_of!(
ZSTDMT_compressionJobFrameHeaderState,
pledged_src_size_plus_one
) == ZSTDMT_HEADER_PLEDGED_OFFSET
);
assert!(
offset_of!(ZSTDMT_compressionJobFrameHeaderState, dict_id)
== ZSTDMT_HEADER_PLEDGED_OFFSET + size_of::<u64>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFrameHeaderState, rep_codes)
== ZSTDMT_HEADER_REP_CODES_OFFSET
);
assert!(
size_of::<ZSTDMT_compressionJobFrameHeaderState>()
== if size_of::<usize>() == 8 { 72 } else { 48 }
);
};
const ZSTD_COMPRESSION_STAGE_CREATED: c_int = 0;
const ZSTD_COMPRESSION_STAGE_INIT: c_int = 1;
const ZSTD_COMPRESSION_STAGE_ONGOING: c_int = 2;
const ZSTDMT_HEADER_PLEDGED_OFFSET: usize = if size_of::<usize>() == 8 {
size_of::<[usize; 6]>()
} else {
size_of::<[usize; 8]>()
};
const ZSTDMT_HEADER_REP_CODES_OFFSET: usize = if size_of::<usize>() == 8 {
size_of::<[usize; 8]>()
} else {
size_of::<[usize; 11]>()
};
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_writeFrameHeader(
state: *const ZSTDMT_compressionJobFrameHeaderState,
) -> usize {
if state.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let state = unsafe { &*state };
if state.stage.is_null() || state.rep_codes.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
if unsafe { *state.stage } == ZSTD_COMPRESSION_STAGE_CREATED {
return ERROR(ZstdErrorCode::StageWrong);
}
let header_size = if unsafe { *state.stage } == ZSTD_COMPRESSION_STAGE_INIT {
let header_size = unsafe {
ZSTD_rust_writeFrameHeader(
state.dst,
state.dst_capacity,
state.no_dict_id_flag,
state.checksum_flag,
state.content_size_flag,
state.format,
state.window_log,
state.pledged_src_size_plus_one.wrapping_sub(1),
state.dict_id,
)
};
if ERR_isError(header_size) {
return header_size;
}
if header_size > state.dst_capacity {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { *state.stage = ZSTD_COMPRESSION_STAGE_ONGOING };
header_size
} else {
0
};
unsafe { ZSTD_rust_invalidateRepCodes(state.rep_codes) };
header_size
}
pub type ZSTDMT_compressionJobStepFn = unsafe extern "C" fn(*mut c_void) -> usize;
pub type ZSTDMT_compressionJobVoidFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_compressionJobCompressFn =
unsafe extern "C" fn(*mut c_void, c_uint) -> ZSTDMT_chunkProcessResult;
pub type ZSTDMT_compressionJobSizeFn = unsafe extern "C" fn(*mut c_void, usize);
/// C-owned leaves for the worker-job finish path.
///
/// Rust owns the error/success branch and callback order. C keeps the job
/// descriptor, pthread objects, private pools, and serial state behind these
/// callbacks; the source size is the only scalar needed for consumed-size
/// publication.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_compressionJobFinishProjection {
pub callbackContext: *mut c_void,
pub srcSize: usize,
pub ensureFinished: Option<ZSTDMT_compressionJobVoidFn>,
pub publishError: Option<ZSTDMT_compressionJobSizeFn>,
pub publishLastBlockSize: Option<ZSTDMT_compressionJobSizeFn>,
pub releaseSeq: Option<ZSTDMT_compressionJobVoidFn>,
pub releaseCCtx: Option<ZSTDMT_compressionJobVoidFn>,
pub publishConsumed: Option<ZSTDMT_compressionJobSizeFn>,
pub signal: Option<ZSTDMT_compressionJobVoidFn>,
}
const _: () = {
assert!(offset_of!(ZSTDMT_compressionJobFinishProjection, callbackContext) == 0);
assert!(offset_of!(ZSTDMT_compressionJobFinishProjection, srcSize) == size_of::<usize>());
assert!(
offset_of!(ZSTDMT_compressionJobFinishProjection, ensureFinished) == 2 * size_of::<usize>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFinishProjection, publishError) == 3 * size_of::<usize>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFinishProjection, publishLastBlockSize)
== 4 * size_of::<usize>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFinishProjection, releaseSeq) == 5 * size_of::<usize>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFinishProjection, releaseCCtx) == 6 * size_of::<usize>()
);
assert!(
offset_of!(ZSTDMT_compressionJobFinishProjection, publishConsumed)
== 7 * size_of::<usize>()
);
assert!(offset_of!(ZSTDMT_compressionJobFinishProjection, signal) == usize::BITS as usize);
assert!(size_of::<ZSTDMT_compressionJobFinishProjection>() == 9 * size_of::<usize>());
};
pub type ZSTDMT_serialWaitForTurnFn = unsafe extern "C" fn(*mut c_void, c_uint) -> c_int;
pub type ZSTDMT_serialGenerateLdmFn =
unsafe extern "C" fn(*mut c_void, *mut ZstdMtRawSeqStore, *const c_void, usize);
pub type ZSTDMT_serialUpdateChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize);
pub type ZSTDMT_serialAdvanceFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateLockFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateWaitFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateBroadcastFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateUnlockFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateCallbackFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateSkipFn = unsafe extern "C" fn(*mut c_void, c_uint, usize);
/// Projection for the MT serial turn wait. Rust owns the lock/wait loop and
/// comparison; C retains the pthread mutex and condition variable behind the
/// callbacks. The lock intentionally remains held when this returns so the
/// caller can perform the serial turn and release it in the advance callback.
#[repr(C)]
pub struct ZSTDMT_RustSerialWaitForTurnState {
callback_context: *mut c_void,
next_job_id: *mut c_uint,
lock: Option<ZSTDMT_serialStateLockFn>,
wait: Option<ZSTDMT_serialStateWaitFn>,
job_id: c_uint,
}
const _: () = {
assert!(size_of::<ZSTDMT_serialStateLockFn>() == size_of::<usize>());
assert!(size_of::<ZSTDMT_serialStateWaitFn>() == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, callback_context) == 0);
assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, next_job_id) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, lock) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, wait) == 3 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, job_id) == 4 * size_of::<usize>());
assert!(size_of::<ZSTDMT_RustSerialWaitForTurnState>() == size_of::<[usize; 5]>());
};
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateWaitForTurn(
state: *const ZSTDMT_RustSerialWaitForTurnState,
) -> c_int {
if state.is_null() {
return 0;
}
let state = unsafe { &*state };
if state.callback_context.is_null() || state.next_job_id.is_null() {
return 0;
}
let (Some(lock), Some(wait)) = (state.lock, state.wait) else {
return 0;
};
unsafe {
lock(state.callback_context);
loop {
if *state.next_job_id >= state.job_id {
break;
}
wait(state.callback_context);
}
c_int::from(*state.next_job_id == state.job_id)
}
}
/// Projection for advancing the MT serial turn. Rust owns the increment and
/// callback order; C retains the condition variable broadcast and mutex
/// unlock operations behind the callbacks.
#[repr(C)]
pub struct ZSTDMT_RustSerialAdvanceState {
callback_context: *mut c_void,
next_job_id: *mut c_uint,
broadcast: Option<ZSTDMT_serialStateBroadcastFn>,
unlock: Option<ZSTDMT_serialStateUnlockFn>,
}
const _: () = {
assert!(size_of::<ZSTDMT_serialStateBroadcastFn>() == size_of::<usize>());
assert!(size_of::<ZSTDMT_serialStateUnlockFn>() == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialAdvanceState, callback_context) == 0);
assert!(offset_of!(ZSTDMT_RustSerialAdvanceState, next_job_id) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialAdvanceState, broadcast) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialAdvanceState, unlock) == 3 * size_of::<usize>());
assert!(size_of::<ZSTDMT_RustSerialAdvanceState>() == size_of::<[usize; 4]>());
};
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateAdvance(
state: *const ZSTDMT_RustSerialAdvanceState,
) {
if state.is_null() {
return;
}
let state = unsafe { &*state };
if state.callback_context.is_null() || state.next_job_id.is_null() {
return;
}
let (Some(broadcast), Some(unlock)) = (state.broadcast, state.unlock) else {
return;
};
unsafe {
*state.next_job_id = (*state.next_job_id).wrapping_add(1);
broadcast(state.callback_context);
unlock(state.callback_context);
}
}
/// Projection for the failed-job serial cleanup path. Rust owns the lock,
/// skip decision, serial-counter publication, and callback ordering; C keeps
/// the mutex, condition variables, and private LDM-window cleanup behind the
/// callbacks.
#[repr(C)]
pub struct ZSTDMT_RustSerialEnsureFinishedState {
callback_context: *mut c_void,
next_job_id: *mut c_uint,
lock: Option<ZSTDMT_serialStateCallbackFn>,
broadcast: Option<ZSTDMT_serialStateCallbackFn>,
ldm_lock: Option<ZSTDMT_serialStateCallbackFn>,
clear_ldm_window: Option<ZSTDMT_serialStateCallbackFn>,
ldm_signal: Option<ZSTDMT_serialStateCallbackFn>,
ldm_unlock: Option<ZSTDMT_serialStateCallbackFn>,
unlock: Option<ZSTDMT_serialStateCallbackFn>,
on_skip: Option<ZSTDMT_serialStateSkipFn>,
c_size: usize,
job_id: c_uint,
}
const _: () = {
assert!(size_of::<ZSTDMT_serialStateCallbackFn>() == size_of::<usize>());
assert!(size_of::<ZSTDMT_serialStateSkipFn>() == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, callback_context) == 0);
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, next_job_id) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, lock) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, broadcast) == 3 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, ldm_lock) == 4 * size_of::<usize>());
assert!(
offset_of!(ZSTDMT_RustSerialEnsureFinishedState, clear_ldm_window)
== 5 * size_of::<usize>()
);
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, ldm_signal) == 6 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, ldm_unlock) == 7 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, unlock) == size_of::<[usize; 8]>());
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, on_skip) == 9 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, c_size) == 10 * size_of::<usize>());
assert!(
offset_of!(ZSTDMT_RustSerialEnsureFinishedState, job_id)
== 10 * size_of::<usize>() + size_of::<usize>()
);
assert!(size_of::<ZSTDMT_RustSerialEnsureFinishedState>() == size_of::<[usize; 12]>());
};
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_serialStateEnsureFinishedResult {
pub skip: c_uint,
pub nextJobID: c_uint,
}
pub type ZSTDMT_waitForJobCompleteFn = unsafe extern "C" fn(*mut c_void, c_uint, c_uint);
pub type ZSTDMT_releaseJobResourceFn = unsafe extern "C" fn(*mut c_void, c_uint);
pub type ZSTDMT_freeCCtxFn = unsafe extern "C" fn(*mut c_void);
/// Projection for MT context teardown. The context and every allocation or
/// synchronization object remain private to C; Rust owns only the ownership
/// branches and teardown ordering between callbacks.
#[repr(C)]
pub struct ZSTDMT_RustFreeCCtxState {
callback_context: *mut c_void,
provided_factory: usize,
round_buffer_present: usize,
free_factory: Option<ZSTDMT_freeCCtxFn>,
release_all_job_resources: Option<ZSTDMT_freeCCtxFn>,
free_jobs: Option<ZSTDMT_freeCCtxFn>,
free_buffer_pool: Option<ZSTDMT_freeCCtxFn>,
free_cctx_pool: Option<ZSTDMT_freeCCtxFn>,
free_seq_pool: Option<ZSTDMT_freeCCtxFn>,
free_serial_state: Option<ZSTDMT_freeCCtxFn>,
free_cdict: Option<ZSTDMT_freeCCtxFn>,
free_round_buffer: Option<ZSTDMT_freeCCtxFn>,
free_mtctx: Option<ZSTDMT_freeCCtxFn>,
}
const _: () = {
assert!(size_of::<ZSTDMT_freeCCtxFn>() == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, callback_context) == 0);
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, provided_factory) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, round_buffer_present) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_factory) == 3 * size_of::<usize>());
assert!(
offset_of!(ZSTDMT_RustFreeCCtxState, release_all_job_resources) == 4 * size_of::<usize>()
);
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_jobs) == 5 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_buffer_pool) == 6 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_cctx_pool) == 7 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_seq_pool) == 8 * (usize::BITS as usize / 8));
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_serial_state) == 9 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_cdict) == 10 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_round_buffer) == 11 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_mtctx) == 12 * size_of::<usize>());
assert!(size_of::<ZSTDMT_RustFreeCCtxState>() == size_of::<[usize; 13]>());
};
pub type ZSTDMT_createCCtxAllocateFn = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
pub type ZSTDMT_createCCtxSetWorkersFn = unsafe extern "C" fn(*mut c_void, c_uint) -> usize;
pub type ZSTDMT_createCCtxSetInitialStateFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_createCCtxFactoryFn = unsafe extern "C" fn(*mut c_void, c_uint) -> *mut c_void;
pub type ZSTDMT_createCCtxJobsFn = unsafe extern "C" fn(*mut c_void, *mut c_uint) -> *mut c_void;
pub type ZSTDMT_createCCtxResourceFn = unsafe extern "C" fn(*mut c_void, c_uint) -> *mut c_void;
pub type ZSTDMT_createCCtxSerialInitFn = unsafe extern "C" fn(*mut c_void) -> c_int;
pub type ZSTDMT_createCCtxFreeFn = unsafe extern "C" fn(*mut c_void);
/// Scalar inputs and callbacks for MT context construction. Rust owns the
/// validation, worker clamp, and resource-construction order; C retains the
/// private context layout, allocator calls, pools, and synchronization.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_RustCreateCCtxProjection {
callback_context: *mut c_void,
requested_nb_workers: c_uint,
max_nb_workers: c_uint,
context_size: usize,
custom_mem: ZstdCustomMem,
}
const _: () = {
assert!(offset_of!(ZSTDMT_RustCreateCCtxProjection, callback_context) == 0);
assert!(
offset_of!(ZSTDMT_RustCreateCCtxProjection, requested_nb_workers) == size_of::<usize>()
);
assert!(
offset_of!(ZSTDMT_RustCreateCCtxProjection, max_nb_workers)
== size_of::<usize>() + size_of::<c_uint>()
);
assert!(
offset_of!(ZSTDMT_RustCreateCCtxProjection, context_size)
== if size_of::<usize>() == 8 { 16 } else { 12 }
);
assert!(
offset_of!(ZSTDMT_RustCreateCCtxProjection, custom_mem)
== if size_of::<usize>() == 8 { 24 } else { 16 }
);
assert!(
size_of::<ZSTDMT_RustCreateCCtxProjection>()
== if size_of::<usize>() == 8 { 48 } else { 28 }
);
};
fn custom_mem_is_valid(custom_mem: ZstdCustomMem) -> bool {
custom_mem.customAlloc.is_some() == custom_mem.customFree.is_some()
}
/// Construct an MT context through the original C resource callbacks.
/// Callbacks are all invoked through the allocation and cleanup sequence even
/// after an intermediate resource returns NULL, matching the C constructor's
/// cleanup contract for partially initialized contexts.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_createCCtx(
projection: *const ZSTDMT_RustCreateCCtxProjection,
allocate: Option<ZSTDMT_createCCtxAllocateFn>,
set_workers: Option<ZSTDMT_createCCtxSetWorkersFn>,
set_initial_state: Option<ZSTDMT_createCCtxSetInitialStateFn>,
create_factory: Option<ZSTDMT_createCCtxFactoryFn>,
create_jobs: Option<ZSTDMT_createCCtxJobsFn>,
create_buffer_pool: Option<ZSTDMT_createCCtxResourceFn>,
create_cctx_pool: Option<ZSTDMT_createCCtxResourceFn>,
create_seq_pool: Option<ZSTDMT_createCCtxResourceFn>,
init_serial: Option<ZSTDMT_createCCtxSerialInitFn>,
free_context: Option<ZSTDMT_createCCtxFreeFn>,
) -> *mut c_void {
let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
return ptr::null_mut();
};
let (
Some(allocate),
Some(set_workers),
Some(set_initial_state),
Some(create_factory),
Some(create_jobs),
Some(create_buffer_pool),
Some(create_cctx_pool),
Some(create_seq_pool),
Some(init_serial),
Some(free_context),
) = (
allocate,
set_workers,
set_initial_state,
create_factory,
create_jobs,
create_buffer_pool,
create_cctx_pool,
create_seq_pool,
init_serial,
free_context,
)
else {
return ptr::null_mut();
};
if projection.callback_context.is_null()
|| projection.requested_nb_workers == 0
|| projection.max_nb_workers == 0
|| projection.context_size == 0
|| !custom_mem_is_valid(projection.custom_mem)
{
return ptr::null_mut();
}
let nb_workers = projection
.requested_nb_workers
.min(projection.max_nb_workers);
let mut nb_jobs = projection.requested_nb_workers.wrapping_add(2);
let context = unsafe { allocate(projection.callback_context, projection.context_size) };
if context.is_null() {
return ptr::null_mut();
}
// The C constructor ignores this setter's result for a valid worker
// count; preserve that exact policy before publishing the custom memory.
unsafe {
let _ = set_workers(projection.callback_context, nb_workers);
set_initial_state(projection.callback_context);
}
let factory = unsafe { create_factory(projection.callback_context, nb_workers) };
let jobs = unsafe { create_jobs(projection.callback_context, &mut nb_jobs) };
let buffer_pool = unsafe { create_buffer_pool(projection.callback_context, nb_workers) };
let cctx_pool = unsafe { create_cctx_pool(projection.callback_context, nb_workers) };
let seq_pool = unsafe { create_seq_pool(projection.callback_context, nb_workers) };
let serial_error = unsafe { init_serial(projection.callback_context) };
if factory.is_null()
|| jobs.is_null()
|| buffer_pool.is_null()
|| cctx_pool.is_null()
|| seq_pool.is_null()
|| serial_error != 0
{
unsafe { free_context(projection.callback_context) };
return ptr::null_mut();
}
context
}
pub type ZSTDMT_resizeStepFn = unsafe extern "C" fn(*mut c_void, c_uint) -> usize;
pub type ZSTDMT_resizeResourceFn = unsafe extern "C" fn(*mut c_void, c_uint) -> *mut c_void;
/// Scalar inputs for MT pool resizing. Rust owns the resize order and
/// short-circuit policy; C retains every private pool and context mutation
/// behind these callbacks.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_RustResizeProjection {
callback_context: *mut c_void,
nb_workers: c_uint,
}
const _: () = {
assert!(offset_of!(ZSTDMT_RustResizeProjection, callback_context) == 0);
assert!(offset_of!(ZSTDMT_RustResizeProjection, nb_workers) == size_of::<usize>());
assert!(
size_of::<ZSTDMT_RustResizeProjection>() == if size_of::<usize>() == 8 { 16 } else { 8 }
);
};
/// Resize MT resources in the original order: worker factory, jobs table,
/// buffer pool, CCtx pool, sequence pool, then the parameter publication.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_resize(
projection: *const ZSTDMT_RustResizeProjection,
resize_factory: Option<ZSTDMT_resizeStepFn>,
expand_jobs: Option<ZSTDMT_resizeStepFn>,
expand_buffer_pool: Option<ZSTDMT_resizeResourceFn>,
expand_cctx_pool: Option<ZSTDMT_resizeResourceFn>,
expand_seq_pool: Option<ZSTDMT_resizeResourceFn>,
set_nb_workers: Option<ZSTDMT_resizeStepFn>,
) -> usize {
let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
return ERROR(ZstdErrorCode::Generic);
};
let (
Some(resize_factory),
Some(expand_jobs),
Some(expand_buffer_pool),
Some(expand_cctx_pool),
Some(expand_seq_pool),
Some(set_nb_workers),
) = (
resize_factory,
expand_jobs,
expand_buffer_pool,
expand_cctx_pool,
expand_seq_pool,
set_nb_workers,
)
else {
return ERROR(ZstdErrorCode::Generic);
};
if projection.callback_context.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let context = projection.callback_context;
let workers = projection.nb_workers;
let error = unsafe { resize_factory(context, workers) };
if ERR_isError(error) {
return error;
}
let error = unsafe { expand_jobs(context, workers) };
if ERR_isError(error) {
return error;
}
if unsafe { expand_buffer_pool(context, workers) }.is_null() {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
if unsafe { expand_cctx_pool(context, workers) }.is_null() {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
if unsafe { expand_seq_pool(context, workers) }.is_null() {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
unsafe { set_nb_workers(context, workers) }
}
type ZSTDMT_waitForLdmLockFn = unsafe extern "C" fn(*mut c_void);
type ZSTDMT_waitForLdmOverlapFn = unsafe extern "C" fn(*mut c_void, *mut c_void, usize) -> c_int;
type ZSTDMT_waitForLdmWaitFn = unsafe extern "C" fn(*mut c_void);
type ZSTDMT_waitForLdmUnlockFn = unsafe extern "C" fn(*mut c_void);
/// Projection for the MT wait that protects reusable input against LDM use.
///
/// Rust owns the enabled branch and lock/overlap/wait ordering. C retains the
/// pthread mutex, condition variable, and private LDM window state.
#[repr(C)]
pub struct ZSTDMT_RustWaitForLdmState {
callback_context: *mut c_void,
buffer_start: *mut c_void,
buffer_capacity: usize,
ldm_enabled: c_int,
lock: Option<ZSTDMT_waitForLdmLockFn>,
overlaps: Option<ZSTDMT_waitForLdmOverlapFn>,
wait: Option<ZSTDMT_waitForLdmWaitFn>,
unlock: Option<ZSTDMT_waitForLdmUnlockFn>,
}
const _: () = {
assert!(size_of::<ZSTDMT_waitForLdmLockFn>() == size_of::<usize>());
assert!(size_of::<ZSTDMT_waitForLdmOverlapFn>() == size_of::<usize>());
assert!(size_of::<ZSTDMT_waitForLdmWaitFn>() == size_of::<usize>());
assert!(size_of::<ZSTDMT_waitForLdmUnlockFn>() == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustWaitForLdmState, callback_context) == 0);
assert!(offset_of!(ZSTDMT_RustWaitForLdmState, buffer_start) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustWaitForLdmState, buffer_capacity) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustWaitForLdmState, ldm_enabled) == 3 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustWaitForLdmState, lock) == 4 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustWaitForLdmState, overlaps) == 5 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustWaitForLdmState, wait) == 6 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustWaitForLdmState, unlock) == 7 * size_of::<usize>());
assert!(size_of::<ZSTDMT_RustWaitForLdmState>() == size_of::<[usize; 8]>());
};
/// Wait until the reusable input range no longer overlaps the serial LDM
/// window, preserving the C synchronization state behind callbacks.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_waitForLdmComplete(state: *const ZSTDMT_RustWaitForLdmState) {
if state.is_null() {
return;
}
let state = unsafe { &*state };
if state.ldm_enabled == 0 || state.callback_context.is_null() {
return;
}
let (Some(lock), Some(overlaps), Some(wait), Some(unlock)) =
(state.lock, state.overlaps, state.wait, state.unlock)
else {
return;
};
unsafe {
lock(state.callback_context);
while overlaps(
state.callback_context,
state.buffer_start,
state.buffer_capacity,
) != 0
{
wait(state.callback_context);
}
unlock(state.callback_context);
}
}
/// Scalar inputs for the MT streaming initializer. The full parameter
/// object, dictionary handles, pools, buffers, and synchronization remain
/// private to C. Rust owns the order in which the C callbacks are invoked and
/// the pure normalization/sizing decisions between those callbacks.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_initCStreamProjection {
pub requestedNbWorkers: c_uint,
pub currentNbWorkers: c_uint,
pub jobSize: usize,
pub jobSizeMin: usize,
pub jobSizeMax: usize,
pub enableLdm: c_int,
pub windowLog: c_uint,
pub chainLog: c_uint,
pub strategy: c_int,
pub overlapLog: c_int,
pub rsyncable: c_int,
pub roundBuffCapacity: usize,
pub allJobsCompleted: c_uint,
}
pub type ZSTDMT_initResizeFn = unsafe extern "C" fn(*mut c_void, c_uint) -> usize;
pub type ZSTDMT_initDrainFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_initApplyParametersFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_initDictionaryFn = unsafe extern "C" fn(*mut c_void) -> usize;
pub type ZSTDMT_initSetSizeFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_initSetRsyncFn = unsafe extern "C" fn(*mut c_void, u64, u64);
pub type ZSTDMT_initSetBufferSizeFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_initResizeRoundBufferFn = unsafe extern "C" fn(*mut c_void, usize) -> usize;
pub type ZSTDMT_initResetStreamFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_initSerialResetFn = unsafe extern "C" fn(*mut c_void, usize) -> usize;
type ZSTDMT_resetStreamCallbackFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialResetVoidFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialResetSetNbSeqFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_serialResetResizeFn = unsafe extern "C" fn(*mut c_void) -> c_int;
pub type ZSTDMT_serialStateVoidFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateInitFn = unsafe extern "C" fn(*mut c_void) -> c_int;
/// Callback projection for the MT stream reset. C retains the private buffer,
/// job, flag, and progress fields; Rust owns the order in which each group is
/// reset.
#[repr(C)]
pub struct ZSTDMT_RustResetStreamState {
callback_context: *mut c_void,
reset_round_buffer: Option<ZSTDMT_resetStreamCallbackFn>,
reset_input_buffer: Option<ZSTDMT_resetStreamCallbackFn>,
reset_job_ids: Option<ZSTDMT_resetStreamCallbackFn>,
reset_frame_flags: Option<ZSTDMT_resetStreamCallbackFn>,
reset_progress: Option<ZSTDMT_resetStreamCallbackFn>,
}
const _: () = {
assert!(size_of::<ZSTDMT_resetStreamCallbackFn>() == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustResetStreamState, callback_context) == 0);
assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_round_buffer) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_input_buffer) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_job_ids) == 3 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_frame_flags) == 4 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_progress) == 5 * size_of::<usize>());
assert!(size_of::<ZSTDMT_RustResetStreamState>() == 6 * size_of::<usize>());
};
/// Reset the MT stream state in the same order as the original C function.
/// Each callback keeps the corresponding private C layout and sentinel values
/// out of the Rust ABI.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_resetStream(state: *const ZSTDMT_RustResetStreamState) {
let Some(state) = (unsafe { state.as_ref() }) else {
return;
};
if state.callback_context.is_null() {
return;
}
let (
Some(reset_round_buffer),
Some(reset_input_buffer),
Some(reset_job_ids),
Some(reset_frame_flags),
Some(reset_progress),
) = (
state.reset_round_buffer,
state.reset_input_buffer,
state.reset_job_ids,
state.reset_frame_flags,
state.reset_progress,
)
else {
return;
};
unsafe {
reset_round_buffer(state.callback_context);
reset_input_buffer(state.callback_context);
reset_job_ids(state.callback_context);
reset_frame_flags(state.callback_context);
reset_progress(state.callback_context);
}
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_flushPublicationResult {
pub status: c_int,
pub toFlush: usize,
pub outputPos: usize,
pub dstFlushed: usize,
}
/// Scalar snapshot of the oldest active C-owned job used by the flush state
/// machine. C keeps the descriptor, mutex, buffer pool, and checksum state;
/// Rust decides when this snapshot can be published and completed.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_flushJobProjection {
pub consumed: usize,
pub cSize: usize,
pub srcSize: usize,
pub dstStart: *const c_void,
pub dstCapacity: usize,
pub dstFlushed: usize,
pub frameChecksumNeeded: c_uint,
}
/// Narrow projection for serializing the C-owned frame checksum. C retains
/// the serial XXH64 state and computes the checksum; Rust owns only the
/// deterministic little-endian write and destination policy.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_frameChecksumProjection {
pub dst: *mut c_void,
pub dstCapacity: usize,
pub cSize: usize,
pub checksum: c_uint,
}
const _: () = {
assert!(offset_of!(ZSTDMT_frameChecksumProjection, dst) == 0);
assert!(offset_of!(ZSTDMT_frameChecksumProjection, dstCapacity) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_frameChecksumProjection, cSize) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_frameChecksumProjection, checksum) == 3 * size_of::<usize>());
assert!(size_of::<c_uint>() == 4);
assert!(size_of::<ZSTDMT_frameChecksumProjection>() == 4 * size_of::<usize>());
};
#[inline]
fn write_frame_checksum(projection: &ZSTDMT_frameChecksumProjection) -> usize {
if projection.dst.is_null()
|| projection.dstCapacity < 4
|| projection.cSize > projection.dstCapacity - 4
{
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe {
MEM_writeLE32(
projection.dst.cast::<u8>().add(projection.cSize).cast(),
projection.checksum,
);
}
projection.cSize + 4
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_writeFrameChecksum(
projection: *const ZSTDMT_frameChecksumProjection,
) -> usize {
let Some(projection) = (unsafe { projection.as_ref() }) else {
return ERROR(ZstdErrorCode::Generic);
};
write_frame_checksum(projection)
}
/// Scalar view of the C-owned MT state needed after the current job is
/// published. The C adapter applies descriptor and context mutations made by
/// Rust's decisions through callbacks.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_flushContextProjection {
pub doneJobID: c_uint,
pub nextJobID: c_uint,
pub jobIDMask: c_uint,
pub jobReady: c_uint,
pub frameEnded: c_uint,
pub inBuffFilled: usize,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_flushProducedResult {
pub result: usize,
pub outputPos: usize,
pub updateAllJobsCompleted: c_uint,
pub allJobsCompleted: c_uint,
}
pub type ZSTDMT_flushProjectJobFn = unsafe extern "C" fn(
opaque: *mut c_void,
job_id: c_uint,
block_to_flush: c_uint,
projection: *mut ZSTDMT_flushJobProjection,
);
pub type ZSTDMT_flushChecksumFn = unsafe extern "C" fn(
opaque: *mut c_void,
job_id: c_uint,
projection: *mut ZSTDMT_flushJobProjection,
);
pub type ZSTDMT_flushUpdateJobFn =
unsafe extern "C" fn(opaque: *mut c_void, job_id: c_uint, dst_flushed: usize);
pub type ZSTDMT_flushCompleteJobFn =
unsafe extern "C" fn(opaque: *mut c_void, job_id: c_uint, src_size: usize, c_size: usize);
pub type ZSTDMT_flushErrorFn = unsafe extern "C" fn(opaque: *mut c_void);
/// Scalar MT state used by the outer streaming scheduler. The C context,
/// input-range ownership, and synchronization objects remain private to C;
/// Rust only reads the fields needed to choose the next operation.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressStreamContextProjection {
pub frameEnded: c_uint,
pub jobReady: c_uint,
pub inBuffStart: *mut c_void,
pub inBuffCapacity: usize,
pub inBuffFilled: usize,
pub targetSectionSize: usize,
pub rsyncable: c_int,
pub rsyncPrimePower: u64,
pub rsyncHitMask: u64,
}
/// Scalar view of the reusable C-owned input range returned by the existing
/// MT input-range seam.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamInputRangeProjection {
pub bufferStart: *mut c_void,
pub bufferCapacity: usize,
pub bufferFilled: usize,
}
/// Explicit projections of the public stream buffers. These deliberately do
/// not use the private C context layout or rely on a Rust view of `ZSTD_inBuffer`
/// and `ZSTD_outBuffer`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamInputProjection {
pub src: *const c_void,
pub size: usize,
pub pos: usize,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamOutputProjection {
pub dst: *mut c_void,
pub size: usize,
pub pos: usize,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_syncPointProjection {
pub toLoad: usize,
pub flush: c_int,
}
/// Result returned by the C-owned pending-output seam after it has updated a
/// temporary output projection.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamFlushResult {
pub result: usize,
pub outputPos: usize,
}
/// Result of one outer MT stream scheduling pass.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressStreamResult {
pub result: usize,
pub inputPos: usize,
pub outputPos: usize,
pub inBuffFilled: usize,
}
const _: () = {
assert!(offset_of!(ZSTDMT_compressStreamResult, result) == 0);
assert!(offset_of!(ZSTDMT_compressStreamResult, inputPos) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_compressStreamResult, outputPos) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_compressStreamResult, inBuffFilled) == 3 * size_of::<usize>());
assert!(size_of::<ZSTDMT_compressStreamResult>() == 4 * size_of::<usize>());
};
pub type ZSTDMT_streamTryGetInputRangeFn = unsafe extern "C" fn(
opaque: *mut c_void,
projection: *mut ZSTDMT_streamInputRangeProjection,
) -> c_int;
pub type ZSTDMT_streamCreateJobFn =
unsafe extern "C" fn(opaque: *mut c_void, src_size: usize, end: c_uint) -> usize;
pub type ZSTDMT_streamFlushProducedFn = unsafe extern "C" fn(
opaque: *mut c_void,
output_dst: *mut c_void,
output_size: usize,
output_pos: usize,
block_to_flush: c_uint,
end: c_uint,
) -> ZSTDMT_streamFlushResult;
const ZSTD_E_END: c_uint = 2;
const ZSTD_E_FLUSH: c_uint = 1;
const ZSTD_E_CONTINUE: c_uint = 0;
/// Scalar view of the C-owned job state needed to finish a streaming frame
/// with an empty last block. The job descriptor, buffer pool, and mutex stay
/// private to C; Rust receives only this projection and a buffer callback.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_emptyBlockJobProjection {
pub lastJob: c_uint,
pub srcSize: usize,
pub firstJob: c_uint,
pub dstStart: *mut c_void,
pub dstCapacity: usize,
pub consumed: usize,
}
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZSTDMT_emptyBlockResult {
pub buffer: ZstdMtBuffer,
pub cSize: usize,
pub clearSource: c_uint,
}
pub type ZSTDMT_bufferGetFn = unsafe extern "C" fn(*mut c_void) -> ZstdMtBuffer;
/// Scalar snapshot of one C-owned job descriptor.
///
/// The C adapter fills the complete projection while holding the descriptor
/// mutex. Rust owns only the ring-scan decisions below; the descriptor layout,
/// synchronization objects, and all pointer ownership stay in C.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_jobProjection {
pub consumed: usize,
pub cSize: usize,
pub srcStart: *const c_void,
pub srcSize: usize,
pub prefixStart: *const c_void,
pub prefixSize: usize,
pub dstFlushed: usize,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_inputRange {
pub start: *const c_void,
pub size: usize,
}
/// Scalar input-buffer state used by the MT range-selection policy. C keeps
/// the input and round-buffer structs private; Rust decides only whether the
/// next target section can be reused and where that section starts.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_tryGetInputRangeProjection {
pub roundBufferStart: *const c_void,
pub roundBufferCapacity: usize,
pub roundBufferPos: usize,
pub prefixStart: *const c_void,
pub prefixSize: usize,
pub targetSectionSize: usize,
pub inUseStart: *const c_void,
pub inUseSize: usize,
}
/// Result of the MT input-range selection. C applies the returned decision,
/// including synchronization waits, prefix copying, and private field
/// updates.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_tryGetInputRangeResult {
pub ready: c_uint,
pub movePrefix: c_uint,
pub bufferStart: *mut c_void,
pub bufferCapacity: usize,
pub roundBufferPos: usize,
}
pub type ZSTDMT_jobProjectionFn = unsafe extern "C" fn(
opaque: *mut c_void,
job_id: c_uint,
projection: *mut ZSTDMT_jobProjection,
);
const FLUSH_PUBLICATION_OK: c_int = 0;
const FLUSH_PUBLICATION_INVALID_BOUNDS: c_int = 1;
#[inline]
fn empty_block_allocation_error() -> ZSTDMT_emptyBlockResult {
ZSTDMT_emptyBlockResult {
buffer: ZstdMtBuffer::default(),
cSize: ERROR(ZstdErrorCode::MemoryAllocation),
clearSource: 0,
}
}
#[inline]
unsafe fn write_last_empty_block_with<F>(
projection: ZSTDMT_emptyBlockJobProjection,
mut get_buffer: F,
) -> ZSTDMT_emptyBlockResult
where
F: FnMut() -> ZstdMtBuffer,
{
debug_assert_eq!(projection.lastJob, 1);
debug_assert_eq!(projection.srcSize, 0);
debug_assert_eq!(projection.firstJob, 0);
debug_assert!(projection.dstStart.is_null());
debug_assert_eq!(projection.dstCapacity, 0);
debug_assert_eq!(projection.consumed, 0);
let buffer = get_buffer();
if buffer.start.is_null() {
return empty_block_allocation_error();
}
let c_size = unsafe {
crate::zstd_compress_frame::ZSTD_writeLastEmptyBlock(buffer.start, buffer.capacity)
};
ZSTDMT_emptyBlockResult {
buffer,
cSize: c_size,
/* The C implementation clears src immediately after acquiring the
* buffer, before serializing the block. */
clearSource: 1,
}
}
/// Acquire and serialize the terminal empty block for a C-owned MT job.
///
/// C keeps the private descriptor and applies the returned state after this
/// callback-driven operation. In particular, an allocation failure leaves
/// the source range untouched just as the original C helper did.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_writeLastEmptyBlock(
projection: *const ZSTDMT_emptyBlockJobProjection,
opaque: *mut c_void,
getBuffer: Option<ZSTDMT_bufferGetFn>,
) -> ZSTDMT_emptyBlockResult {
let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
return empty_block_allocation_error();
};
let Some(get_buffer) = getBuffer else {
return empty_block_allocation_error();
};
unsafe { write_last_empty_block_with(projection, || get_buffer(opaque)) }
}
const CREATE_JOB_TABLE_FULL: c_uint = 0;
const CREATE_JOB_POST: c_uint = 1;
const CREATE_JOB_EMPTY: c_uint = 2;
/// Scalar MT state supplied by C for one compression-job creation attempt.
///
/// The job descriptor, input-buffer ownership, and synchronization objects
/// remain private to C. Rust owns the ring-capacity and scheduling policy over
/// this scalar view.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_createJobProjection {
pub doneJobID: c_uint,
pub nextJobID: c_uint,
pub jobIDMask: c_uint,
pub jobReady: c_uint,
pub srcStart: *const c_void,
pub srcSize: usize,
pub inBuffFilled: usize,
pub prefixStart: *const c_void,
pub prefixSize: usize,
pub targetPrefixSize: usize,
pub endFrame: c_uint,
pub checksumFlag: c_uint,
}
/// The scalar fields needed by C to initialize its private job descriptor and
/// the surrounding input state after Rust chooses to prepare a new job.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_jobInitialization {
pub srcStart: *const c_void,
pub srcSize: usize,
pub prefixStart: *const c_void,
pub prefixSize: usize,
pub nextPrefixStart: *const c_void,
pub nextPrefixSize: usize,
pub roundBuffPosDelta: usize,
pub jobNumber: c_uint,
pub firstJob: c_uint,
pub lastJob: c_uint,
pub frameChecksumNeeded: c_uint,
pub clearChecksumFlag: c_uint,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_createJobResult {
pub returnCode: usize,
pub action: c_uint,
pub jobID: c_uint,
pub jobNumber: c_uint,
pub nextJobID: c_uint,
pub jobReady: c_uint,
}
pub type ZSTDMT_prepareJobFn = unsafe extern "C" fn(
opaque: *mut c_void,
job_id: c_uint,
initialization: *const ZSTDMT_jobInitialization,
);
pub type ZSTDMT_writeEmptyJobFn = unsafe extern "C" fn(opaque: *mut c_void, job_id: c_uint);
pub type ZSTDMT_tryAddJobFn = unsafe extern "C" fn(opaque: *mut c_void, job_id: c_uint) -> c_int;
#[inline]
fn create_job_table_full(projection: ZSTDMT_createJobProjection) -> ZSTDMT_createJobResult {
ZSTDMT_createJobResult {
action: CREATE_JOB_TABLE_FULL,
jobID: projection.nextJobID & projection.jobIDMask,
jobNumber: projection.nextJobID,
nextJobID: projection.nextJobID,
jobReady: projection.jobReady,
..ZSTDMT_createJobResult::default()
}
}
#[inline]
fn create_job_initialization(
projection: ZSTDMT_createJobProjection,
) -> (c_uint, ZSTDMT_jobInitialization) {
let job_number = projection.nextJobID;
let first_job = (job_number == 0) as c_uint;
let last_job = (projection.endFrame != 0) as c_uint;
let frame_checksum_needed =
(projection.checksumFlag != 0 && last_job != 0 && job_number > 0) as c_uint;
let (next_prefix_start, next_prefix_size) = if last_job != 0 {
(ptr::null(), 0)
} else {
let next_prefix_size = projection.srcSize.min(projection.targetPrefixSize);
let next_prefix_start = projection
.srcStart
.cast::<u8>()
.wrapping_add(projection.srcSize - next_prefix_size)
.cast();
(next_prefix_start, next_prefix_size)
};
(
projection.nextJobID & projection.jobIDMask,
ZSTDMT_jobInitialization {
srcStart: projection.srcStart,
srcSize: projection.srcSize,
prefixStart: projection.prefixStart,
prefixSize: projection.prefixSize,
nextPrefixStart: next_prefix_start,
nextPrefixSize: next_prefix_size,
roundBuffPosDelta: projection.srcSize,
jobNumber: job_number,
firstJob: first_job,
lastJob: last_job,
frameChecksumNeeded: frame_checksum_needed,
clearChecksumFlag: (last_job != 0 && first_job != 0) as c_uint,
},
)
}
#[inline]
fn create_compression_job_with<P, E, T>(
projection: ZSTDMT_createJobProjection,
mut prepare_job: P,
mut write_empty_job: E,
mut try_add_job: T,
) -> ZSTDMT_createJobResult
where
P: FnMut(c_uint, &ZSTDMT_jobInitialization),
E: FnMut(c_uint),
T: FnMut(c_uint) -> bool,
{
/* Match the original unsigned C comparison, including wraparound. */
if projection.nextJobID > projection.doneJobID.wrapping_add(projection.jobIDMask) {
return create_job_table_full(projection);
}
let job_id = projection.nextJobID & projection.jobIDMask;
let mut result = ZSTDMT_createJobResult {
action: CREATE_JOB_POST,
jobID: job_id,
jobNumber: projection.nextJobID,
nextJobID: projection.nextJobID,
jobReady: projection.jobReady,
..ZSTDMT_createJobResult::default()
};
if projection.jobReady == 0 {
debug_assert!(projection.inBuffFilled >= projection.srcSize);
let (_, initialization) = create_job_initialization(projection);
prepare_job(job_id, &initialization);
/* A non-first empty job is represented by the terminal empty block,
* not by a worker submission. */
if projection.srcSize == 0 && projection.nextJobID > 0 {
write_empty_job(job_id);
result.action = CREATE_JOB_EMPTY;
result.nextJobID = projection.nextJobID.wrapping_add(1);
result.jobReady = 0;
return result;
}
}
if try_add_job(job_id) {
result.nextJobID = projection.nextJobID.wrapping_add(1);
result.jobReady = 0;
} else {
result.jobReady = 1;
}
result
}
/// Apply MT scalar scheduling policy while C retains all private descriptor,
/// pool, mutex, condition-variable, and worker-callback operations.
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_createCompressionJob(
projection: *const ZSTDMT_createJobProjection,
opaque: *mut c_void,
prepareJob: Option<ZSTDMT_prepareJobFn>,
writeEmptyJob: Option<ZSTDMT_writeEmptyJobFn>,
tryAddJob: Option<ZSTDMT_tryAddJobFn>,
) -> ZSTDMT_createJobResult {
let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
return ZSTDMT_createJobResult::default();
};
let (Some(prepare_job), Some(write_empty_job), Some(try_add_job)) =
(prepareJob, writeEmptyJob, tryAddJob)
else {
return ZSTDMT_createJobResult::default();
};
create_compression_job_with(
projection,
|job_id, initialization| unsafe {
prepare_job(opaque, job_id, initialization);
},
|job_id| unsafe {
write_empty_job(opaque, job_id);
},
|job_id| unsafe { try_add_job(opaque, job_id) != 0 },
)
}
/// Finish a worker job with Rust-owned branch and callback ordering.
///
/// Error publication happens before serial completion and resource release,
/// matching the original C path so failed jobs advance the serial state with
/// the published error. Success publishes the final block size only after the
/// private sequence and CCtx callbacks return. Both paths publish the full
/// consumed size and signal only after all preceding leaves have run.
#[inline]
fn finish_compression_job_with<E, F, R, C, B, P, S>(
error: usize,
last_block_size: usize,
src_size: usize,
mut publish_error: E,
mut ensure_finished: F,
mut release_seq: R,
mut release_cctx: C,
mut publish_last_block_size: B,
mut publish_consumed: P,
mut signal: S,
) where
E: FnMut(usize),
F: FnMut(),
R: FnMut(),
C: FnMut(),
B: FnMut(usize),
P: FnMut(usize),
S: FnMut(),
{
let failed = ERR_isError(error);
if failed {
publish_error(error);
}
ensure_finished();
release_seq();
release_cctx();
if !failed {
publish_last_block_size(last_block_size);
}
publish_consumed(src_size);
signal();
}
/// Run the high-level worker-job sequence while C owns all codec and
/// synchronization operations behind callbacks.
///
/// Resource acquisition and every codec-facing stage can fail with a zstd
/// error. Rust stops at the first such error, normalizes the final block size,
/// then applies the C-owned finish leaves through the Rust-owned policy above.
#[inline]
fn compression_job_with<A, P, S, B, Q, H, C, T>(
projection: ZSTDMT_compressionJobProjection,
finish: ZSTDMT_compressionJobFinishProjection,
mut acquire_resources: A,
mut prepare_parameters: P,
mut generate_sequences: S,
mut begin_job: B,
mut apply_sequences: Q,
mut write_frame_header: H,
mut compress_job: C,
mut trace_job: T,
) where
A: FnMut() -> usize,
P: FnMut(),
S: FnMut(),
B: FnMut() -> usize,
Q: FnMut(),
H: FnMut() -> usize,
C: FnMut(c_uint) -> ZSTDMT_chunkProcessResult,
T: FnMut(),
{
let (
Some(ensure_finished),
Some(publish_error),
Some(publish_last_block_size),
Some(release_seq),
Some(release_cctx),
Some(publish_consumed),
Some(signal),
) = (
finish.ensureFinished,
finish.publishError,
finish.publishLastBlockSize,
finish.releaseSeq,
finish.releaseCCtx,
finish.publishConsumed,
finish.signal,
)
else {
return;
};
let mut error = acquire_resources();
let mut last_block_size = 0;
if !ERR_isError(error) {
prepare_parameters();
generate_sequences();
error = begin_job();
}
if !ERR_isError(error) {
apply_sequences();
if projection.firstJob == 0 {
error = write_frame_header();
}
}
if !ERR_isError(error) {
let result = compress_job(projection.lastJob);
if ERR_isError(result.error) {
error = result.error;
} else {
last_block_size = result.lastBlockSize;
trace_job();
}
}
if ERR_isError(error) {
last_block_size = 0;
}
finish_compression_job_with(
error,
last_block_size,
finish.srcSize,
|error| unsafe { publish_error(finish.callbackContext, error) },
|| unsafe { ensure_finished(finish.callbackContext) },
|| unsafe { release_seq(finish.callbackContext) },
|| unsafe { release_cctx(finish.callbackContext) },
|last_block_size| unsafe {
publish_last_block_size(finish.callbackContext, last_block_size)
},
|src_size| unsafe { publish_consumed(finish.callbackContext, src_size) },
|| unsafe { signal(finish.callbackContext) },
);
}
/// Drive one ordered MT serial-state turn.
///
/// The wait callback returns with the C-owned serial mutex held. Rust then
/// performs the LDM-before-checksum policy only for the current job and always
/// invokes the advance callback exactly once, including when an earlier job
/// has already skipped this turn.
#[inline]
fn serial_state_gen_sequences_with<W, L, C, A>(
seq_store: &mut ZstdMtRawSeqStore,
src: *const c_void,
src_size: usize,
job_id: c_uint,
ldm_enabled: bool,
checksum_enabled: bool,
mut wait_for_turn: W,
mut generate_ldm: L,
mut update_checksum: C,
mut advance: A,
) where
W: FnMut(c_uint) -> bool,
L: FnMut(&mut ZstdMtRawSeqStore, *const c_void, usize),
C: FnMut(*const c_void, usize),
A: FnMut(),
{
if wait_for_turn(job_id) {
if ldm_enabled {
generate_ldm(seq_store, src, src_size);
}
if checksum_enabled && src_size != 0 {
update_checksum(src, src_size);
}
}
advance();
}
/// Own MT serial turn/skip and LDM/checksum ordering while C retains the
/// mutexes, LDM window/hash state, checksum state, and raw sequence storage.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateGenSequences(
seq_store: *mut ZstdMtRawSeqStore,
src: *const c_void,
src_size: usize,
job_id: c_uint,
ldm_enabled: c_int,
checksum_enabled: c_int,
opaque: *mut c_void,
wait_for_turn: Option<ZSTDMT_serialWaitForTurnFn>,
generate_ldm: Option<ZSTDMT_serialGenerateLdmFn>,
update_checksum: Option<ZSTDMT_serialUpdateChecksumFn>,
advance: Option<ZSTDMT_serialAdvanceFn>,
) {
let (Some(wait_for_turn), Some(generate_ldm), Some(update_checksum), Some(advance)) =
(wait_for_turn, generate_ldm, update_checksum, advance)
else {
return;
};
if seq_store.is_null() {
return;
}
unsafe {
serial_state_gen_sequences_with(
&mut *seq_store,
src,
src_size,
job_id,
ldm_enabled != 0,
checksum_enabled != 0,
|job_id| wait_for_turn(opaque, job_id) != 0,
|seq_store, src, src_size| generate_ldm(opaque, seq_store, src, src_size),
|src, src_size| update_checksum(opaque, src, src_size),
|| advance(opaque),
);
}
}
/// Decide whether a failed job must advance the shared serial turn.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_serialStateEnsureFinished(
next_job_id: c_uint,
job_id: c_uint,
) -> ZSTDMT_serialStateEnsureFinishedResult {
if next_job_id <= job_id {
ZSTDMT_serialStateEnsureFinishedResult {
skip: 1,
nextJobID: job_id.wrapping_add(1),
}
} else {
ZSTDMT_serialStateEnsureFinishedResult {
skip: 0,
nextJobID: next_job_id,
}
}
}
/// Own the failed-job serial cleanup order while C retains synchronization and
/// private LDM-window operations behind callbacks.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateEnsureFinishedOrchestrated(
state: *const ZSTDMT_RustSerialEnsureFinishedState,
) {
if state.is_null() {
return;
}
let state = unsafe { &*state };
if state.callback_context.is_null() || state.next_job_id.is_null() {
return;
}
let (
Some(lock),
Some(broadcast),
Some(ldm_lock),
Some(clear_ldm_window),
Some(ldm_signal),
Some(ldm_unlock),
Some(unlock),
Some(on_skip),
) = (
state.lock,
state.broadcast,
state.ldm_lock,
state.clear_ldm_window,
state.ldm_signal,
state.ldm_unlock,
state.unlock,
state.on_skip,
)
else {
return;
};
unsafe {
lock(state.callback_context);
let result = ZSTDMT_rust_serialStateEnsureFinished(*state.next_job_id, state.job_id);
if result.skip != 0 {
on_skip(state.callback_context, state.job_id, state.c_size);
*state.next_job_id = result.nextJobID;
broadcast(state.callback_context);
ldm_lock(state.callback_context);
clear_ldm_window(state.callback_context);
ldm_signal(state.callback_context);
ldm_unlock(state.callback_context);
}
unlock(state.callback_context);
}
}
/// Wait for each submitted MT job in ring order. C retains the per-job
/// mutex, condition variable, and consumed/source counters behind one
/// callback; Rust owns the ring-slot and completion progression policy.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_waitForAllJobsCompleted(
mut done_job_id: c_uint,
next_job_id: c_uint,
job_id_mask: c_uint,
opaque: *mut c_void,
wait_for_job: Option<ZSTDMT_waitForJobCompleteFn>,
) -> c_uint {
let Some(wait_for_job) = wait_for_job else {
return done_job_id;
};
while done_job_id < next_job_id {
unsafe { wait_for_job(opaque, done_job_id & job_id_mask, done_job_id) };
done_job_id = done_job_id.wrapping_add(1);
}
done_job_id
}
/// Visit every MT job slot so C can release its private descriptor resources.
/// Rust owns the deterministic inclusive ring-table iteration; C retains the
/// buffer-pool, mutex/condition, and descriptor cleanup side effects.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_releaseAllJobResources(
job_id_mask: c_uint,
opaque: *mut c_void,
release_job: Option<ZSTDMT_releaseJobResourceFn>,
) {
let Some(release_job) = release_job else {
return;
};
for job_id in 0..=job_id_mask {
unsafe { release_job(opaque, job_id) };
}
}
/// C ABI entry point for the worker-job orchestration. C supplies callbacks
/// that keep the private descriptor, pools, mutexes, and codec operations on
/// the C side of this narrow projection.
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressionJob(
projection: *const ZSTDMT_compressionJobProjection,
finishProjection: *const ZSTDMT_compressionJobFinishProjection,
opaque: *mut c_void,
acquireResources: Option<ZSTDMT_compressionJobStepFn>,
prepareParameters: Option<ZSTDMT_compressionJobVoidFn>,
generateSequences: Option<ZSTDMT_compressionJobVoidFn>,
beginJob: Option<ZSTDMT_compressionJobStepFn>,
applySequences: Option<ZSTDMT_compressionJobVoidFn>,
compressJob: Option<ZSTDMT_compressionJobCompressFn>,
traceJob: Option<ZSTDMT_compressionJobVoidFn>,
) {
let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
return;
};
let Some(finish) = (unsafe { finishProjection.as_ref() }).copied() else {
return;
};
let (
Some(acquire_resources),
Some(prepare_parameters),
Some(generate_sequences),
Some(begin_job),
Some(apply_sequences),
Some(compress_job),
Some(trace_job),
) = (
acquireResources,
prepareParameters,
generateSequences,
beginJob,
applySequences,
compressJob,
traceJob,
)
else {
return;
};
compression_job_with(
projection,
finish,
|| unsafe { acquire_resources(opaque) },
|| unsafe { prepare_parameters(opaque) },
|| unsafe { generate_sequences(opaque) },
|| unsafe { begin_job(opaque) },
|| unsafe { apply_sequences(opaque) },
|| unsafe { ZSTDMT_rust_writeFrameHeader(projection.frameHeaderState) },
|last_job| unsafe { compress_job(opaque, last_job) },
|| unsafe { trace_job(opaque) },
);
}
#[inline]
fn normalize_mt_job_size(job_size: usize, job_size_min: usize, job_size_max: usize) -> usize {
if job_size != 0 && job_size < job_size_min {
job_size_min
} else if job_size > job_size_max {
job_size_max
} else {
job_size
}
}
#[inline]
fn init_round_buffer_capacity(
projection: ZSTDMT_initCStreamProjection,
target_prefix_size: usize,
target_section_size: usize,
) -> usize {
let window_size = if projection.enableLdm == ZSTD_PS_ENABLE {
1usize << projection.windowLog as usize
} else {
0
};
let nb_slack_buffers = 2 + usize::from(target_prefix_size > 0);
let slack_size = target_section_size.wrapping_mul(nb_slack_buffers);
let nb_workers = projection.requestedNbWorkers.max(1) as usize;
let sections_size = target_section_size.wrapping_mul(nb_workers);
window_size.max(sections_size).wrapping_add(slack_size)
}
#[inline]
fn init_rsync_parameters(target_section_size: usize) -> (u64, u64) {
let job_size_kb = (target_section_size >> 10) as u32;
debug_assert!(job_size_kb >= 1);
let rsync_bits = ZSTD_highbit32(job_size_kb) + 10;
debug_assert!(rsync_bits >= (RSYNC_MIN_BLOCK_LOG + 2) as u32);
let hit_mask = 1u64.wrapping_shl(rsync_bits).wrapping_sub(1);
(hit_mask, rolling_hash_prime_power(RSYNC_LENGTH as c_uint))
}
/// Run the high-level MT streaming initialization policy while C retains all
/// private state and side effects behind callbacks. The callback order is
/// intentionally the order of the original C initializer: resize, normalize,
/// drain, attach the first dictionary, size buffers, reset stream state,
/// update the active dictionary, and finally reset serial state.
#[inline]
fn init_c_stream_with<
Resize,
Drain,
ApplyParams,
PrepareDictionary,
SetPrefixSize,
SetSectionSize,
SetRsync,
SetBufferSize,
ResizeRoundBuffer,
ResetStream,
UpdateDictionary,
SerialReset,
>(
projection: ZSTDMT_initCStreamProjection,
mut resize: Resize,
mut drain: Drain,
mut apply_parameters: ApplyParams,
mut prepare_dictionary: PrepareDictionary,
mut set_target_prefix_size: SetPrefixSize,
mut set_target_section_size: SetSectionSize,
mut set_rsync: SetRsync,
mut set_buffer_size: SetBufferSize,
mut resize_round_buffer: ResizeRoundBuffer,
mut reset_stream: ResetStream,
mut update_dictionary: UpdateDictionary,
mut serial_reset: SerialReset,
) -> usize
where
Resize: FnMut(c_uint) -> usize,
Drain: FnMut(),
ApplyParams: FnMut(usize),
PrepareDictionary: FnMut() -> usize,
SetPrefixSize: FnMut(usize),
SetSectionSize: FnMut(usize),
SetRsync: FnMut(u64, u64),
SetBufferSize: FnMut(usize),
ResizeRoundBuffer: FnMut(usize) -> usize,
ResetStream: FnMut(),
UpdateDictionary: FnMut() -> usize,
SerialReset: FnMut(usize) -> usize,
{
if projection.requestedNbWorkers != projection.currentNbWorkers {
let error = resize(projection.requestedNbWorkers);
if ERR_isError(error) {
return error;
}
}
let job_size = normalize_mt_job_size(
projection.jobSize,
projection.jobSizeMin,
projection.jobSizeMax,
);
if projection.allJobsCompleted == 0 {
drain();
}
apply_parameters(job_size);
let error = prepare_dictionary();
if ERR_isError(error) {
return error;
}
let target_prefix_size = compute_overlap_size(
projection.windowLog,
projection.chainLog,
projection.strategy,
projection.overlapLog,
projection.enableLdm,
);
set_target_prefix_size(target_prefix_size);
let initial_target_section_size = if job_size == 0 {
1usize
<< compute_target_job_log(
projection.windowLog,
projection.chainLog,
projection.strategy,
projection.enableLdm,
) as usize
} else {
job_size
};
debug_assert!(initial_target_section_size <= projection.jobSizeMax);
set_target_section_size(initial_target_section_size);
if projection.rsyncable != 0 {
let (hit_mask, prime_power) = init_rsync_parameters(initial_target_section_size);
set_rsync(hit_mask, prime_power);
}
let target_section_size = initial_target_section_size.max(target_prefix_size);
if target_section_size != initial_target_section_size {
set_target_section_size(target_section_size);
}
set_buffer_size(crate::zstd_compress_api::ZSTD_compressBound(
target_section_size,
));
let round_capacity =
init_round_buffer_capacity(projection, target_prefix_size, target_section_size);
if projection.roundBuffCapacity < round_capacity {
let error = resize_round_buffer(round_capacity);
if ERR_isError(error) {
return error;
}
}
reset_stream();
let error = update_dictionary();
if ERR_isError(error) {
return error;
}
serial_reset(target_section_size)
}
/// Run the MT serial-state reset policy while C retains its private LDM
/// tables, dictionary/window state, checksum state, and allocator callbacks.
/// The callback order mirrors the original reset path and stops immediately
/// when table allocation reports failure.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateReset(
enable_ldm: c_int,
checksum_enabled: c_int,
max_nb_seq: usize,
opaque: *mut c_void,
reset_next_job: Option<ZSTDMT_serialResetVoidFn>,
reset_checksum: Option<ZSTDMT_serialResetVoidFn>,
set_nb_seq: Option<ZSTDMT_serialResetSetNbSeqFn>,
reset_window: Option<ZSTDMT_serialResetVoidFn>,
resize_tables: Option<ZSTDMT_serialResetResizeFn>,
zero_tables: Option<ZSTDMT_serialResetVoidFn>,
load_dictionary: Option<ZSTDMT_serialResetVoidFn>,
copy_window: Option<ZSTDMT_serialResetVoidFn>,
) -> c_int {
let (
Some(reset_next_job),
Some(reset_checksum),
Some(set_nb_seq),
Some(reset_window),
Some(resize_tables),
Some(zero_tables),
Some(load_dictionary),
Some(copy_window),
) = (
reset_next_job,
reset_checksum,
set_nb_seq,
reset_window,
resize_tables,
zero_tables,
load_dictionary,
copy_window,
)
else {
return 1;
};
unsafe {
reset_next_job(opaque);
if checksum_enabled != 0 {
reset_checksum(opaque);
}
if enable_ldm == ZSTD_PS_ENABLE {
set_nb_seq(opaque, max_nb_seq);
reset_window(opaque);
if resize_tables(opaque) != 0 {
return 1;
}
zero_tables(opaque);
load_dictionary(opaque);
copy_window(opaque);
}
}
0
}
/// Run the MT serial-state synchronization initialization order and preserve
/// the original bitwise error aggregation. C owns each pthread primitive and
/// the zeroing callback.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateInit(
opaque: *mut c_void,
zero_state: Option<ZSTDMT_serialStateVoidFn>,
init_mutex: Option<ZSTDMT_serialStateInitFn>,
init_cond: Option<ZSTDMT_serialStateInitFn>,
init_ldm_mutex: Option<ZSTDMT_serialStateInitFn>,
init_ldm_cond: Option<ZSTDMT_serialStateInitFn>,
) -> c_int {
let (
Some(zero_state),
Some(init_mutex),
Some(init_cond),
Some(init_ldm_mutex),
Some(init_ldm_cond),
) = (
zero_state,
init_mutex,
init_cond,
init_ldm_mutex,
init_ldm_cond,
)
else {
return 1;
};
unsafe {
zero_state(opaque);
init_mutex(opaque) | init_cond(opaque) | init_ldm_mutex(opaque) | init_ldm_cond(opaque)
}
}
/// Run the MT serial-state synchronization destruction and table-free order.
/// The callbacks retain ownership of platform synchronization and allocator
/// details on the C side.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateFree(
opaque: *mut c_void,
destroy_mutex: Option<ZSTDMT_serialStateVoidFn>,
destroy_cond: Option<ZSTDMT_serialStateVoidFn>,
destroy_ldm_mutex: Option<ZSTDMT_serialStateVoidFn>,
destroy_ldm_cond: Option<ZSTDMT_serialStateVoidFn>,
free_tables: Option<ZSTDMT_serialStateVoidFn>,
) {
let (
Some(destroy_mutex),
Some(destroy_cond),
Some(destroy_ldm_mutex),
Some(destroy_ldm_cond),
Some(free_tables),
) = (
destroy_mutex,
destroy_cond,
destroy_ldm_mutex,
destroy_ldm_cond,
free_tables,
)
else {
return;
};
unsafe {
destroy_mutex(opaque);
destroy_cond(opaque);
destroy_ldm_mutex(opaque);
destroy_ldm_cond(opaque);
free_tables(opaque);
}
}
/// Run the MT context teardown order while C callbacks retain all private
/// layouts, allocators, worker pools, and synchronization details.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_freeCCtx(state: *const ZSTDMT_RustFreeCCtxState) -> usize {
if state.is_null() {
return 0;
}
let state = unsafe { &*state };
let (
Some(free_factory),
Some(release_all_job_resources),
Some(free_jobs),
Some(free_buffer_pool),
Some(free_cctx_pool),
Some(free_seq_pool),
Some(free_serial_state),
Some(free_cdict),
Some(free_round_buffer),
Some(free_mtctx),
) = (
state.free_factory,
state.release_all_job_resources,
state.free_jobs,
state.free_buffer_pool,
state.free_cctx_pool,
state.free_seq_pool,
state.free_serial_state,
state.free_cdict,
state.free_round_buffer,
state.free_mtctx,
)
else {
return 0;
};
unsafe {
if state.provided_factory == 0 {
free_factory(state.callback_context);
}
release_all_job_resources(state.callback_context);
free_jobs(state.callback_context);
free_buffer_pool(state.callback_context);
free_cctx_pool(state.callback_context);
free_seq_pool(state.callback_context);
free_serial_state(state.callback_context);
free_cdict(state.callback_context);
if state.round_buffer_present != 0 {
free_round_buffer(state.callback_context);
}
free_mtctx(state.callback_context);
}
0
}
/// C ABI entry point for the MT streaming initializer. C owns every
/// allocation, dictionary handle, synchronization object, and private context
/// mutation; this wrapper only connects those operations to the Rust policy.
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_initCStream(
projection: *const ZSTDMT_initCStreamProjection,
opaque: *mut c_void,
resize: Option<ZSTDMT_initResizeFn>,
drain: Option<ZSTDMT_initDrainFn>,
applyParameters: Option<ZSTDMT_initApplyParametersFn>,
prepareDictionary: Option<ZSTDMT_initDictionaryFn>,
setTargetPrefixSize: Option<ZSTDMT_initSetSizeFn>,
setTargetSectionSize: Option<ZSTDMT_initSetSizeFn>,
setRsync: Option<ZSTDMT_initSetRsyncFn>,
setBufferSize: Option<ZSTDMT_initSetBufferSizeFn>,
resizeRoundBuffer: Option<ZSTDMT_initResizeRoundBufferFn>,
resetStream: Option<ZSTDMT_initResetStreamFn>,
updateDictionary: Option<ZSTDMT_initDictionaryFn>,
serialReset: Option<ZSTDMT_initSerialResetFn>,
) -> usize {
let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
return ERROR(ZstdErrorCode::Generic);
};
let (
Some(resize),
Some(drain),
Some(apply_parameters),
Some(prepare_dictionary),
Some(set_target_prefix_size),
Some(set_target_section_size),
Some(set_rsync),
Some(set_buffer_size),
Some(resize_round_buffer),
Some(reset_stream),
Some(update_dictionary),
Some(serial_reset),
) = (
resize,
drain,
applyParameters,
prepareDictionary,
setTargetPrefixSize,
setTargetSectionSize,
setRsync,
setBufferSize,
resizeRoundBuffer,
resetStream,
updateDictionary,
serialReset,
)
else {
return ERROR(ZstdErrorCode::Generic);
};
init_c_stream_with(
projection,
|nb_workers| unsafe { resize(opaque, nb_workers) },
|| unsafe { drain(opaque) },
|job_size| unsafe { apply_parameters(opaque, job_size) },
|| unsafe { prepare_dictionary(opaque) },
|size| unsafe { set_target_prefix_size(opaque, size) },
|size| unsafe { set_target_section_size(opaque, size) },
|hit_mask, prime_power| unsafe { set_rsync(opaque, hit_mask, prime_power) },
|size| unsafe { set_buffer_size(opaque, size) },
|capacity| unsafe { resize_round_buffer(opaque, capacity) },
|| unsafe { reset_stream(opaque) },
|| unsafe { update_dictionary(opaque) },
|target_section_size| unsafe { serial_reset(opaque, target_section_size) },
)
}
#[inline]
fn invalid_flush_publication(
output_pos: usize,
dst_flushed: usize,
) -> ZSTDMT_flushPublicationResult {
ZSTDMT_flushPublicationResult {
status: FLUSH_PUBLICATION_INVALID_BOUNDS,
toFlush: 0,
outputPos: output_pos,
dstFlushed: dst_flushed,
}
}
#[inline]
unsafe fn publish_job_output_with(
output_dst: *mut c_void,
output_size: usize,
output_pos: usize,
job_dst: *const c_void,
job_capacity: usize,
c_size: usize,
dst_flushed: usize,
) -> ZSTDMT_flushPublicationResult {
let Some(output_available) = output_size.checked_sub(output_pos) else {
return invalid_flush_publication(output_pos, dst_flushed);
};
let Some(job_available) = c_size.checked_sub(dst_flushed) else {
return invalid_flush_publication(output_pos, dst_flushed);
};
if c_size > job_capacity {
return invalid_flush_publication(output_pos, dst_flushed);
}
let to_flush = output_available.min(job_available);
if to_flush > 0 {
if output_dst.is_null() || job_dst.is_null() {
return invalid_flush_publication(output_pos, dst_flushed);
}
unsafe {
ptr::copy_nonoverlapping(
job_dst.cast::<u8>().add(dst_flushed),
output_dst.cast::<u8>().add(output_pos),
to_flush,
);
}
}
ZSTDMT_flushPublicationResult {
status: FLUSH_PUBLICATION_OK,
toFlush: to_flush,
outputPos: output_pos + to_flush,
dstFlushed: dst_flushed + to_flush,
}
}
#[inline]
fn flush_produced_result(result: usize, output_pos: usize) -> ZSTDMT_flushProducedResult {
ZSTDMT_flushProducedResult {
result,
outputPos: output_pos,
updateAllJobsCompleted: 0,
allJobsCompleted: 0,
}
}
#[inline]
unsafe fn flush_produced_with<P, C, U, F, E>(
context: ZSTDMT_flushContextProjection,
output_dst: *mut c_void,
output_size: usize,
output_pos: usize,
block_to_flush: c_uint,
end: c_uint,
mut project_job: P,
mut add_frame_checksum: C,
mut update_job: U,
mut complete_job: F,
mut on_error: E,
) -> ZSTDMT_flushProducedResult
where
P: FnMut(c_uint, c_uint) -> ZSTDMT_flushJobProjection,
C: FnMut(c_uint, &mut ZSTDMT_flushJobProjection),
U: FnMut(c_uint, usize),
F: FnMut(c_uint, usize, usize),
E: FnMut(),
{
let mut output_pos = output_pos;
let mut done_job_id = context.doneJobID;
if done_job_id < context.nextJobID {
let job_id = done_job_id & context.jobIDMask;
let mut projection = project_job(job_id, block_to_flush);
let src_consumed = projection.consumed;
let src_size = projection.srcSize;
if ERR_isError(projection.cSize) {
on_error();
return flush_produced_result(projection.cSize, output_pos);
}
/* The checksum is added only once, after the worker has consumed the
* whole source range. C retains the serial XXH state and updates the
* projected cSize through this callback. */
if src_consumed == src_size && projection.frameChecksumNeeded != 0 {
add_frame_checksum(job_id, &mut projection);
}
if ERR_isError(projection.cSize) {
on_error();
return flush_produced_result(projection.cSize, output_pos);
}
let c_size = projection.cSize;
let mut dst_flushed = projection.dstFlushed;
if c_size > 0 {
let publication = unsafe {
publish_job_output_with(
output_dst,
output_size,
output_pos,
projection.dstStart,
projection.dstCapacity,
c_size,
dst_flushed,
)
};
if publication.status != FLUSH_PUBLICATION_OK {
return flush_produced_result(ERROR(ZstdErrorCode::Generic), output_pos);
}
output_pos = publication.outputPos;
dst_flushed = publication.dstFlushed;
update_job(job_id, dst_flushed);
if src_consumed == src_size && dst_flushed == c_size {
complete_job(job_id, src_size, c_size);
done_job_id = done_job_id.wrapping_add(1);
}
}
/* Match the C API's distinction between known pending output and a
* live worker whose output size is not known yet. */
if c_size > dst_flushed {
return flush_produced_result(c_size - dst_flushed, output_pos);
}
if src_size > src_consumed {
return flush_produced_result(1, output_pos);
}
}
if done_job_id < context.nextJobID {
return flush_produced_result(1, output_pos);
}
if context.jobReady != 0 {
return flush_produced_result(1, output_pos);
}
if context.inBuffFilled > 0 {
return flush_produced_result(1, output_pos);
}
/* All jobs are entirely flushed. For ZSTD_e_end the caller asks whether
* the frame itself has ended, rather than whether buffers are empty. */
ZSTDMT_flushProducedResult {
result: if end == ZSTD_E_END {
(context.frameEnded == 0) as usize
} else {
0
},
outputPos: output_pos,
updateAllJobsCompleted: 1,
allJobsCompleted: context.frameEnded,
}
}
#[cfg(not(test))]
unsafe extern "C" {
fn ZSTD_compressContinue_public(
cctx: *mut c_void,
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
) -> usize;
fn ZSTD_compressEnd_public(
cctx: *mut c_void,
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
) -> usize;
}
#[inline]
unsafe fn compress_job_chunks_with(
cctx: *mut c_void,
src: *const c_void,
src_size: usize,
dst: *mut c_void,
dst_capacity: usize,
chunk_size: usize,
last_job: c_uint,
progress_context: *mut c_void,
progress_callback: Option<ZSTDMT_chunkProgressFn>,
compress_continue: ZstdMtCompressFn,
compress_end: ZstdMtCompressFn,
) -> ZSTDMT_chunkProcessResult {
debug_assert!(chunk_size > 0);
debug_assert!(chunk_size.is_power_of_two());
let nb_chunks = src_size.div_ceil(chunk_size);
let mut input = src.cast::<u8>();
let mut produced = 0usize;
for chunk_number in 1..nb_chunks {
let c_size = unsafe {
compress_continue(
cctx,
dst.cast::<u8>().wrapping_add(produced).cast(),
dst_capacity.wrapping_sub(produced),
input.cast(),
chunk_size,
)
};
if ERR_isError(c_size) {
return ZSTDMT_chunkProcessResult {
error: c_size,
lastBlockSize: 0,
};
}
input = input.wrapping_add(chunk_size);
produced = produced.wrapping_add(c_size);
debug_assert!(produced < dst_capacity);
if let Some(callback) = progress_callback {
unsafe { callback(progress_context, c_size, chunk_size * chunk_number) };
}
}
if nb_chunks > 0 || last_job != 0 {
let last_block_size1 = src_size & (chunk_size - 1);
let last_block_size = if last_block_size1 == 0 && src_size >= chunk_size {
chunk_size
} else {
last_block_size1
};
let c_size = unsafe {
let dst = dst.cast::<u8>().wrapping_add(produced).cast();
let dst_capacity = dst_capacity.wrapping_sub(produced);
let compress = if last_job != 0 {
compress_end
} else {
compress_continue
};
compress(cctx, dst, dst_capacity, input.cast(), last_block_size)
};
if ERR_isError(c_size) {
return ZSTDMT_chunkProcessResult {
error: c_size,
lastBlockSize: 0,
};
}
return ZSTDMT_chunkProcessResult {
error: 0,
lastBlockSize: c_size,
};
}
ZSTDMT_chunkProcessResult {
error: 0,
lastBlockSize: 0,
}
}
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressJobChunks(
cctx: *mut c_void,
src: *const c_void,
src_size: usize,
dst: *mut c_void,
dst_capacity: usize,
chunk_size: usize,
last_job: c_uint,
progress_context: *mut c_void,
progress_callback: Option<ZSTDMT_chunkProgressFn>,
) -> ZSTDMT_chunkProcessResult {
unsafe {
compress_job_chunks_with(
cctx,
src,
src_size,
dst,
dst_capacity,
chunk_size,
last_job,
progress_context,
progress_callback,
ZSTD_compressContinue_public,
ZSTD_compressEnd_public,
)
}
}
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_publishJobOutput(
output_dst: *mut c_void,
output_size: usize,
output_pos: usize,
job_dst: *const c_void,
job_capacity: usize,
c_size: usize,
dst_flushed: usize,
) -> ZSTDMT_flushPublicationResult {
unsafe {
publish_job_output_with(
output_dst,
output_size,
output_pos,
job_dst,
job_capacity,
c_size,
dst_flushed,
)
}
}
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_flushProduced(
context: *const ZSTDMT_flushContextProjection,
output_dst: *mut c_void,
output_size: usize,
output_pos: usize,
block_to_flush: c_uint,
end: c_uint,
opaque: *mut c_void,
projectJob: Option<ZSTDMT_flushProjectJobFn>,
addFrameChecksum: Option<ZSTDMT_flushChecksumFn>,
updateJob: Option<ZSTDMT_flushUpdateJobFn>,
completeJob: Option<ZSTDMT_flushCompleteJobFn>,
onError: Option<ZSTDMT_flushErrorFn>,
) -> ZSTDMT_flushProducedResult {
let Some(context) = (unsafe { context.as_ref() }).copied() else {
return flush_produced_result(ERROR(ZstdErrorCode::Generic), output_pos);
};
let (
Some(project_job),
Some(add_frame_checksum),
Some(update_job),
Some(complete_job),
Some(on_error),
) = (
projectJob,
addFrameChecksum,
updateJob,
completeJob,
onError,
)
else {
return flush_produced_result(ERROR(ZstdErrorCode::Generic), output_pos);
};
unsafe {
flush_produced_with(
context,
output_dst,
output_size,
output_pos,
block_to_flush,
end,
|job_id, block_to_flush| {
let mut projection = ZSTDMT_flushJobProjection::default();
project_job(opaque, job_id, block_to_flush, &mut projection);
projection
},
|job_id, projection| {
add_frame_checksum(opaque, job_id, projection);
},
|job_id, dst_flushed| {
update_job(opaque, job_id, dst_flushed);
},
|job_id, src_size, c_size| {
complete_job(opaque, job_id, src_size, c_size);
},
|| {
on_error(opaque);
},
)
}
}
#[inline]
fn compress_stream_result(
result: usize,
input_pos: usize,
output_pos: usize,
in_buff_filled: usize,
) -> ZSTDMT_compressStreamResult {
ZSTDMT_compressStreamResult {
result,
inputPos: input_pos,
outputPos: output_pos,
inBuffFilled: in_buff_filled,
}
}
/// Run one outer MT streaming pass while C retains the context and worker
/// implementation details.
///
/// The callbacks are intentionally narrow: input-range acquisition exposes a
/// C-owned buffer, job creation can post the prepared job, and flushing can
/// wait on C-owned job synchronization. Rust owns the bounded input copy, the
/// ordering of those operations, the end-directive adjustments, and the public
/// progress/return policy.
#[allow(clippy::too_many_arguments)]
unsafe fn compress_stream_generic_with<T, J, F, S>(
context: ZSTDMT_compressStreamContextProjection,
input: ZSTDMT_streamInputProjection,
output: ZSTDMT_streamOutputProjection,
end_op: c_uint,
mut try_get_input_range: T,
mut create_job: J,
mut flush_produced: F,
mut find_sync_point: S,
) -> ZSTDMT_compressStreamResult
where
T: FnMut(&mut ZSTDMT_streamInputRangeProjection) -> c_int,
J: FnMut(usize, c_uint) -> usize,
F: FnMut(*mut c_void, usize, usize, c_uint, c_uint) -> ZSTDMT_streamFlushResult,
S: FnMut(
&ZSTDMT_compressStreamContextProjection,
&ZSTDMT_streamInputProjection,
&ZSTDMT_streamInputRangeProjection,
) -> ZSTDMT_syncPointProjection,
{
let mut input_pos = input.pos;
let output_pos = output.pos;
if input_pos > input.size
|| output_pos > output.size
|| context.targetSectionSize == 0
|| context.inBuffFilled > context.inBuffCapacity
|| (context.inBuffStart.is_null() && context.inBuffFilled != 0)
{
return compress_stream_result(
ERROR(ZstdErrorCode::Generic),
input_pos,
output_pos,
context.inBuffFilled,
);
}
if context.frameEnded != 0 && end_op == ZSTD_E_CONTINUE {
return compress_stream_result(
ERROR(ZstdErrorCode::StageWrong),
input_pos,
output_pos,
context.inBuffFilled,
);
}
let mut end_op = end_op;
let mut input_range = ZSTDMT_streamInputRangeProjection {
bufferStart: context.inBuffStart,
bufferCapacity: context.inBuffCapacity,
bufferFilled: context.inBuffFilled,
};
let mut forward_input_progress = false;
/* Fill the current round-buffer section unless a prepared job is waiting
* for a worker. This is the same ordering as the C implementation: a
* blocked range acquisition is allowed to fall through to the flush pass.
*/
if context.jobReady == 0 && input.size > input_pos {
if input_range.bufferStart.is_null() {
debug_assert_eq!(input_range.bufferFilled, 0);
let _ = try_get_input_range(&mut input_range);
}
if !input_range.bufferStart.is_null() {
let sync_point = find_sync_point(&context, &input, &input_range);
if sync_point.flush != 0 && end_op == ZSTD_E_CONTINUE {
end_op = ZSTD_E_FLUSH;
}
let available = input.size - input_pos;
let Some(buffer_left) = input_range
.bufferCapacity
.checked_sub(input_range.bufferFilled)
else {
return compress_stream_result(
ERROR(ZstdErrorCode::Generic),
input_pos,
output_pos,
input_range.bufferFilled,
);
};
if sync_point.toLoad > available
|| sync_point.toLoad > buffer_left
|| (sync_point.toLoad != 0 && input.src.is_null())
{
return compress_stream_result(
ERROR(ZstdErrorCode::Generic),
input_pos,
output_pos,
input_range.bufferFilled,
);
}
if sync_point.toLoad != 0 {
let source = input
.src
.cast::<u8>()
.wrapping_add(input_pos)
.cast::<c_void>();
unsafe {
ptr::copy_nonoverlapping(
source.cast::<u8>(),
input_range
.bufferStart
.cast::<u8>()
.add(input_range.bufferFilled),
sync_point.toLoad,
);
}
input_pos += sync_point.toLoad;
input_range.bufferFilled += sync_point.toLoad;
forward_input_progress = true;
}
}
}
if input_pos < input.size && end_op == ZSTD_E_END {
/* The caller cannot end a frame while input remains outside the
* current job. The current pass must flush this job first. */
end_op = ZSTD_E_FLUSH;
}
if context.jobReady != 0
|| input_range.bufferFilled >= context.targetSectionSize
|| (end_op != ZSTD_E_CONTINUE && input_range.bufferFilled > 0)
|| (end_op == ZSTD_E_END && context.frameEnded == 0)
{
debug_assert!(input_range.bufferFilled <= context.targetSectionSize);
let create_result = create_job(input_range.bufferFilled, end_op);
if ERR_isError(create_result) {
return compress_stream_result(
create_result,
input_pos,
output_pos,
input_range.bufferFilled,
);
}
}
let flush_result = flush_produced(
output.dst,
output.size,
output_pos,
u32::from(!forward_input_progress),
end_op,
);
let remaining_to_flush = flush_result.result;
let result = if input_pos < input.size {
remaining_to_flush.max(1)
} else {
remaining_to_flush
};
compress_stream_result(
result,
input_pos,
flush_result.outputPos,
input_range.bufferFilled,
)
}
/// C ABI entry point for the MT outer scheduling/flush policy.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressStreamGeneric(
context: *const ZSTDMT_compressStreamContextProjection,
input: *const ZSTDMT_streamInputProjection,
output: *const ZSTDMT_streamOutputProjection,
end_op: c_uint,
opaque: *mut c_void,
tryGetInputRange: Option<ZSTDMT_streamTryGetInputRangeFn>,
createJob: Option<ZSTDMT_streamCreateJobFn>,
flushProduced: Option<ZSTDMT_streamFlushProducedFn>,
) -> ZSTDMT_compressStreamResult {
let Some(context) = (unsafe { context.as_ref() }).copied() else {
return compress_stream_result(ERROR(ZstdErrorCode::Generic), 0, 0, 0);
};
let Some(input) = (unsafe { input.as_ref() }).copied() else {
return compress_stream_result(ERROR(ZstdErrorCode::Generic), 0, 0, context.inBuffFilled);
};
let Some(output) = (unsafe { output.as_ref() }).copied() else {
return compress_stream_result(
ERROR(ZstdErrorCode::Generic),
input.pos,
0,
context.inBuffFilled,
);
};
let (Some(try_get_input_range), Some(create_job), Some(flush_produced)) =
(tryGetInputRange, createJob, flushProduced)
else {
return compress_stream_result(
ERROR(ZstdErrorCode::Generic),
input.pos,
output.pos,
context.inBuffFilled,
);
};
unsafe {
compress_stream_generic_with(
context,
input,
output,
end_op,
|projection| try_get_input_range(opaque, projection),
|src_size, end| create_job(opaque, src_size, end),
|dst, size, pos, block_to_flush, end| {
flush_produced(opaque, dst, size, pos, block_to_flush, end)
},
|context, input, input_range| {
let (to_load, flush) = find_synchronization_point(
input.src,
input.size,
input.pos,
context.targetSectionSize,
input_range.bufferStart.cast_const(),
input_range.bufferFilled,
context.rsyncable,
context.rsyncPrimePower,
context.rsyncHitMask,
);
ZSTDMT_syncPointProjection {
toLoad: to_load,
flush,
}
},
)
}
}
#[inline]
fn get_input_data_in_use_with<F>(
first_job_id: c_uint,
last_job_id: c_uint,
job_id_mask: c_uint,
round_buffer_capacity: usize,
target_section_size: usize,
mut project_job: F,
) -> ZSTDMT_inputRange
where
F: FnMut(c_uint) -> ZSTDMT_jobProjection,
{
debug_assert!(target_section_size > 0);
if target_section_size == 0 {
return ZSTDMT_inputRange::default();
}
/* No job can refer to the first round buffer before this point. */
let jobs_before_round_buffer_reuse = round_buffer_capacity / target_section_size;
/* C's usual arithmetic conversions promote `lastJobID` to size_t here. */
if (last_job_id as usize) < jobs_before_round_buffer_reuse {
return ZSTDMT_inputRange::default();
}
for job_id in first_job_id..last_job_id {
let projection = project_job(job_id & job_id_mask);
if projection.consumed >= projection.srcSize {
continue;
}
if projection.prefixSize > 0 {
debug_assert!((projection.prefixStart as usize) <= (projection.srcStart as usize));
return ZSTDMT_inputRange {
start: projection.prefixStart,
size: projection.prefixSize,
};
}
return ZSTDMT_inputRange {
start: projection.srcStart,
size: projection.srcSize,
};
}
ZSTDMT_inputRange::default()
}
/// Find the earliest still-in-use input range in the C-owned job ring.
///
/// C supplies each snapshot under its private job mutex. Rust performs the
/// ring ordering and prefix/source selection without receiving the job table
/// or any of its private synchronization layout.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_getInputDataInUse(
firstJobID: c_uint,
lastJobID: c_uint,
jobIDMask: c_uint,
roundBufferCapacity: usize,
targetSectionSize: usize,
opaque: *mut c_void,
projectJob: Option<ZSTDMT_jobProjectionFn>,
) -> ZSTDMT_inputRange {
let Some(project_job) = projectJob else {
return ZSTDMT_inputRange::default();
};
get_input_data_in_use_with(
firstJobID,
lastJobID,
jobIDMask,
roundBufferCapacity,
targetSectionSize,
|job_id| {
let mut projection = ZSTDMT_jobProjection::default();
unsafe { project_job(opaque, job_id, &mut projection) };
projection
},
)
}
#[inline]
fn to_flush_now_with<F>(
done_job_id: c_uint,
next_job_id: c_uint,
job_id_mask: c_uint,
mut project_job: F,
) -> usize
where
F: FnMut(c_uint) -> ZSTDMT_jobProjection,
{
if done_job_id == next_job_id {
return 0;
}
let projection = project_job(done_job_id & job_id_mask);
let produced = if ERR_isError(projection.cSize) {
0
} else {
projection.cSize
};
let flushed = if ERR_isError(projection.cSize) {
0
} else {
projection.dstFlushed
};
debug_assert!(flushed <= produced);
debug_assert!(projection.consumed <= projection.srcSize);
let to_flush = produced.wrapping_sub(flushed);
if to_flush == 0 {
/* A live job with no output yet must still have input remaining. */
debug_assert!(projection.consumed < projection.srcSize);
}
to_flush
}
/// Return the output already produced by the oldest active C job but not yet
/// published to the caller's output buffer.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_toFlushNow(
doneJobID: c_uint,
nextJobID: c_uint,
jobIDMask: c_uint,
opaque: *mut c_void,
projectJob: Option<ZSTDMT_jobProjectionFn>,
) -> usize {
let Some(project_job) = projectJob else {
return 0;
};
to_flush_now_with(doneJobID, nextJobID, jobIDMask, |job_id| {
let mut projection = ZSTDMT_jobProjection::default();
unsafe { project_job(opaque, job_id, &mut projection) };
projection
})
}
#[inline]
fn try_get_input_range_with(
projection: ZSTDMT_tryGetInputRangeProjection,
) -> ZSTDMT_tryGetInputRangeResult {
let invalid = || ZSTDMT_tryGetInputRangeResult::default();
debug_assert!(projection.roundBufferPos <= projection.roundBufferCapacity);
debug_assert!(projection.targetSectionSize > 0);
if projection.roundBufferPos > projection.roundBufferCapacity
|| projection.targetSectionSize == 0
|| projection.targetSectionSize > projection.roundBufferCapacity
{
return invalid();
}
let space_left = projection.roundBufferCapacity - projection.roundBufferPos;
let (round_buffer_pos, move_prefix) = if space_left < projection.targetSectionSize {
/* The C implementation first checks the prefix range at the start of
* the round buffer before resetting the position there. */
if is_overlapped(
projection.roundBufferStart,
projection.prefixSize,
projection.inUseStart,
projection.inUseSize,
) != 0
{
return invalid();
}
if projection.prefixSize > projection.roundBufferCapacity - projection.targetSectionSize {
return invalid();
}
(projection.prefixSize, 1)
} else {
(projection.roundBufferPos, 0)
};
let buffer_start = projection
.roundBufferStart
.cast::<u8>()
.wrapping_add(round_buffer_pos)
.cast_mut()
.cast::<c_void>();
if move_prefix == 0 {
/* This is an internal invariant in the original C helper: a live
* prefix must never be overwritten by the next source section. */
debug_assert_eq!(
is_overlapped(
buffer_start,
projection.targetSectionSize,
projection.prefixStart,
projection.prefixSize,
),
0
);
}
if is_overlapped(
buffer_start,
projection.targetSectionSize,
projection.inUseStart,
projection.inUseSize,
) != 0
{
return invalid();
}
ZSTDMT_tryGetInputRangeResult {
ready: 1,
movePrefix: move_prefix,
bufferStart: buffer_start,
bufferCapacity: projection.targetSectionSize,
roundBufferPos: round_buffer_pos,
}
}
/// Select the next reusable round-buffer section for MT input. The C adapter
/// retains all synchronization, prefix copying, and private context updates.
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_tryGetInputRange(
projection: *const ZSTDMT_tryGetInputRangeProjection,
) -> ZSTDMT_tryGetInputRangeResult {
let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
return ZSTDMT_tryGetInputRangeResult::default();
};
try_get_input_range_with(projection)
}
#[inline]
fn cycle_log(chain_log: c_uint, strategy: c_int) -> c_uint {
chain_log.wrapping_sub((strategy >= ZSTD_BTLAZY2) as c_uint)
}
#[inline]
fn compute_target_job_log(
window_log: c_uint,
chain_log: c_uint,
strategy: c_int,
enable_ldm: c_int,
) -> c_uint {
let job_log = if enable_ldm == ZSTD_PS_ENABLE {
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on cycleLog instead. */
21.max(cycle_log(chain_log, strategy).wrapping_add(3))
} else {
20.max(window_log.wrapping_add(2))
};
job_log.min(ZSTDMT_JOBLOG_MAX)
}
#[inline]
fn overlap_log_default(strategy: c_int) -> c_int {
match strategy {
ZSTD_BTULTRA2 => 9,
ZSTD_BTULTRA | ZSTD_BTOPT => 8,
ZSTD_BTLAZY2 | ZSTD_LAZY2 => 7,
ZSTD_LAZY | ZSTD_GREEDY | ZSTD_DFAST | ZSTD_FAST => 6,
_ => 6,
}
}
#[inline]
fn overlap_log(overlap_log: c_int, strategy: c_int) -> c_int {
debug_assert!((0..=9).contains(&overlap_log));
if overlap_log == 0 {
overlap_log_default(strategy)
} else {
overlap_log
}
}
#[inline]
fn compute_overlap_size(
window_log: c_uint,
chain_log: c_uint,
strategy: c_int,
overlap_log_value: c_int,
enable_ldm: c_int,
) -> usize {
let overlap_r_log = 9 - overlap_log(overlap_log_value, strategy);
let mut overlap_log_value = if overlap_r_log >= 8 {
0
} else {
window_log as c_int - overlap_r_log
};
debug_assert!((0..=8).contains(&overlap_r_log));
if enable_ldm == ZSTD_PS_ENABLE {
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on chainLog instead.
* Then, ovLog becomes a fraction of the jobSize, rather than windowSize */
let target_job_log = compute_target_job_log(window_log, chain_log, strategy, enable_ldm);
overlap_log_value = window_log.min(target_job_log.wrapping_sub(2)) as c_int - overlap_r_log;
}
debug_assert!(overlap_log_value >= 0);
debug_assert!(overlap_log_value <= ZSTD_WINDOWLOG_MAX as c_int);
if overlap_log_value == 0 {
0
} else {
1usize << overlap_log_value as usize
}
}
/// C ABI for the pure MT target job-log policy.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_computeTargetJobLog(
windowLog: c_uint,
chainLog: c_uint,
strategy: c_int,
enableLdm: c_int,
) -> c_uint {
compute_target_job_log(windowLog, chainLog, strategy, enableLdm)
}
/// C ABI for the MT overlap-log default/selection policy.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_overlapLog(overlapLog: c_int, strategy: c_int) -> c_int {
overlap_log(overlapLog, strategy)
}
/// C ABI for the pure MT overlap-size policy.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_computeOverlapSize(
windowLog: c_uint,
chainLog: c_uint,
strategy: c_int,
overlapLog: c_int,
enableLdm: c_int,
) -> usize {
compute_overlap_size(windowLog, chainLog, strategy, overlapLog, enableLdm)
}
#[inline]
fn frame_progression(
consumed: u64,
in_buff_filled: usize,
produced: u64,
current_job_id: c_uint,
) -> ZSTD_frameProgression {
ZSTD_frameProgression {
// C's usual arithmetic conversions promote size_t to U64 before wrapping.
ingested: consumed.wrapping_add(in_buff_filled as u64),
consumed,
produced,
flushed: produced,
currentJobID: current_job_id,
nbActiveWorkers: 0,
}
}
#[inline]
fn frame_progression_add_job(
mut progression: ZSTD_frameProgression,
src_size: usize,
consumed: usize,
produced: usize,
flushed: usize,
) -> ZSTD_frameProgression {
progression.ingested = progression.ingested.wrapping_add(src_size as u64);
progression.consumed = progression.consumed.wrapping_add(consumed as u64);
progression.produced = progression.produced.wrapping_add(produced as u64);
progression.flushed = progression.flushed.wrapping_add(flushed as u64);
progression.nbActiveWorkers = progression
.nbActiveWorkers
.wrapping_add((consumed < src_size) as c_uint);
progression
}
#[inline]
fn frame_progression_with_jobs<F>(
consumed: u64,
in_buff_filled: usize,
produced: u64,
current_job_id: c_uint,
done_job_id: c_uint,
next_job_id: c_uint,
job_ready: c_uint,
job_id_mask: c_uint,
mut project_job: F,
) -> ZSTD_frameProgression
where
F: FnMut(c_uint) -> ZSTDMT_jobProjection,
{
let mut progression = frame_progression(consumed, in_buff_filled, produced, current_job_id);
debug_assert!(job_ready <= 1);
let last_job_id = next_job_id.wrapping_add(job_ready);
for job_id in done_job_id..last_job_id {
let projection = project_job(job_id & job_id_mask);
let (produced, flushed) = if ERR_isError(projection.cSize) {
(0, 0)
} else {
(projection.cSize, projection.dstFlushed)
};
debug_assert!(flushed <= produced);
progression = frame_progression_add_job(
progression,
projection.srcSize,
projection.consumed,
produced,
flushed,
);
}
progression
}
/// Construct the base MT frame progression from C-owned scalar state.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_frameProgression(
consumed: u64,
in_buff_filled: usize,
produced: u64,
current_job_id: c_uint,
) -> ZSTD_frameProgression {
frame_progression(consumed, in_buff_filled, produced, current_job_id)
}
/// Add one C-normalized, mutex-protected job snapshot to MT frame progression.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_frameProgressionAddJob(
progression: ZSTD_frameProgression,
src_size: usize,
consumed: usize,
produced: usize,
flushed: usize,
) -> ZSTD_frameProgression {
frame_progression_add_job(progression, src_size, consumed, produced, flushed)
}
/// Aggregate the base MT progression with mutex-protected C job snapshots.
/// Rust owns only the ring scan and normalization policy; C retains the job
/// descriptor layout and takes each descriptor mutex in `projectJob`.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_frameProgressionWithJobs(
consumed: u64,
in_buff_filled: usize,
produced: u64,
current_job_id: c_uint,
done_job_id: c_uint,
next_job_id: c_uint,
job_ready: c_uint,
job_id_mask: c_uint,
opaque: *mut c_void,
project_job: Option<ZSTDMT_jobProjectionFn>,
) -> ZSTD_frameProgression {
let Some(project_job) = project_job else {
return frame_progression(consumed, in_buff_filled, produced, current_job_id);
};
frame_progression_with_jobs(
consumed,
in_buff_filled,
produced,
current_job_id,
done_job_id,
next_job_id,
job_ready,
job_id_mask,
|job_id| {
let mut projection = ZSTDMT_jobProjection::default();
unsafe { project_job(opaque, job_id, &mut projection) };
projection
},
)
}
#[inline]
unsafe fn rolling_hash_append(mut hash: u64, input: *const u8, size: usize) -> u64 {
for pos in 0..size {
let byte = u64::from(unsafe { *input.wrapping_add(pos) });
hash = hash
.wrapping_mul(PRIME8_BYTES)
.wrapping_add(byte.wrapping_add(ROLL_HASH_CHAR_OFFSET));
}
hash
}
#[inline]
unsafe fn rolling_hash_compute(input: *const u8, size: usize) -> u64 {
unsafe { rolling_hash_append(0, input, size) }
}
#[inline]
fn rolling_hash_rotate(hash: u64, to_remove: u8, to_add: u8, prime_power: u64) -> u64 {
hash.wrapping_sub(
u64::from(to_remove)
.wrapping_add(ROLL_HASH_CHAR_OFFSET)
.wrapping_mul(prime_power),
)
.wrapping_mul(PRIME8_BYTES)
.wrapping_add(u64::from(to_add).wrapping_add(ROLL_HASH_CHAR_OFFSET))
}
#[inline]
fn rolling_hash_prime_power(length: c_uint) -> u64 {
let mut base = PRIME8_BYTES;
let mut exponent = u64::from(length.wrapping_sub(1));
let mut power: u64 = 1;
while exponent != 0 {
if exponent & 1 != 0 {
power = power.wrapping_mul(base);
}
exponent >>= 1;
base = base.wrapping_mul(base);
}
power
}
/// C ABI for the rolling-hash prime power used by rsyncable MT compression.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_rollingHashPrimePower(length: c_uint) -> u64 {
rolling_hash_prime_power(length)
}
/// Search for a synchronization point without exposing the C MT context or
/// its private buffer types across the FFI boundary.
#[inline]
unsafe fn find_synchronization_point(
input_src: *const c_void,
input_size: usize,
input_pos: usize,
target_section_size: usize,
in_buff_start: *const c_void,
in_buff_filled: usize,
rsyncable: c_int,
prime_power: u64,
hit_mask: u64,
) -> (usize, c_int) {
let istart = input_src.cast::<u8>().wrapping_add(input_pos);
let available = input_size.wrapping_sub(input_pos);
let mut to_load = available.min(target_section_size.wrapping_sub(in_buff_filled));
let mut flush = 0;
if rsyncable == 0 {
return (to_load, flush);
}
if in_buff_filled.wrapping_add(available) < RSYNC_MIN_BLOCK_SIZE {
return (to_load, flush);
}
if in_buff_filled.wrapping_add(to_load) < RSYNC_LENGTH {
return (to_load, flush);
}
let (mut pos, prev, mut hash) = if in_buff_filled < RSYNC_MIN_BLOCK_SIZE {
let pos = RSYNC_MIN_BLOCK_SIZE.wrapping_sub(in_buff_filled);
if pos >= RSYNC_LENGTH {
let prev = istart.wrapping_add(pos.wrapping_sub(RSYNC_LENGTH));
let hash = unsafe { rolling_hash_compute(prev, RSYNC_LENGTH) };
(pos, prev, hash)
} else {
debug_assert!(in_buff_filled >= RSYNC_LENGTH);
let prev = in_buff_start
.cast::<u8>()
.wrapping_add(in_buff_filled.wrapping_sub(RSYNC_LENGTH));
let hash = unsafe { rolling_hash_compute(prev.wrapping_add(pos), RSYNC_LENGTH - pos) };
let hash = unsafe { rolling_hash_append(hash, istart, pos) };
(pos, prev, hash)
}
} else {
let pos = 0;
let prev = in_buff_start
.cast::<u8>()
.wrapping_add(in_buff_filled.wrapping_sub(RSYNC_LENGTH));
let hash = unsafe { rolling_hash_compute(prev, RSYNC_LENGTH) };
if (hash & hit_mask) == hit_mask {
return (0, 1);
}
(pos, prev, hash)
};
debug_assert!(
pos < RSYNC_LENGTH
|| unsafe {
rolling_hash_compute(
istart.wrapping_add(pos.wrapping_sub(RSYNC_LENGTH)),
RSYNC_LENGTH,
) == hash
}
);
while pos < to_load {
let to_remove = if pos < RSYNC_LENGTH {
unsafe { *prev.wrapping_add(pos) }
} else {
unsafe { *istart.wrapping_add(pos.wrapping_sub(RSYNC_LENGTH)) }
};
let to_add = unsafe { *istart.wrapping_add(pos) };
hash = rolling_hash_rotate(hash, to_remove, to_add, prime_power);
debug_assert!(in_buff_filled.wrapping_add(pos) >= RSYNC_MIN_BLOCK_SIZE);
if (hash & hit_mask) == hit_mask {
to_load = pos + 1;
flush = 1;
pos += 1;
break;
}
pos += 1;
}
debug_assert!(
pos < RSYNC_LENGTH
|| unsafe {
rolling_hash_compute(
istart.wrapping_add(pos.wrapping_sub(RSYNC_LENGTH)),
RSYNC_LENGTH,
) == hash
}
);
(to_load, flush)
}
/// C ABI for the pure MT synchronization-point policy.
///
/// The caller retains ownership of the MT context, input position, and
/// buffer updates. Rust only scans the two byte ranges and writes the two
/// scalar results.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_findSynchronizationPoint(
inputSrc: *const c_void,
inputSize: usize,
inputPos: usize,
targetSectionSize: usize,
inBuffStart: *const c_void,
inBuffFilled: usize,
rsyncable: c_int,
primePower: u64,
hitMask: u64,
toLoad: *mut usize,
flush: *mut c_int,
) {
let (computed_to_load, computed_flush) = unsafe {
find_synchronization_point(
inputSrc,
inputSize,
inputPos,
targetSectionSize,
inBuffStart,
inBuffFilled,
rsyncable,
primePower,
hitMask,
)
};
unsafe {
*toLoad = computed_to_load;
*flush = computed_flush;
}
}
type ZstdAllocFunction = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
type ZstdFreeFunction = unsafe extern "C" fn(*mut c_void, *mut c_void);
/// ABI-compatible representation of `ZSTD_customMem` from `zstd.h`.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZstdCustomMem {
pub customAlloc: Option<ZstdAllocFunction>,
pub customFree: Option<ZstdFreeFunction>,
pub opaque: *mut c_void,
}
/// ABI-compatible buffer returned to the C adapter.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZstdMtBuffer {
pub start: *mut c_void,
pub capacity: usize,
}
/// ABI-compatible representation of the private C `ZSTD_window_t` used by
/// the MT LDM overlap callback.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZstdMtWindow {
pub nextSrc: *const u8,
pub base: *const u8,
pub dictBase: *const u8,
pub dictLimit: u32,
pub lowLimit: u32,
pub nbOverflowCorrections: u32,
}
const _: () = {
assert!(std::mem::offset_of!(ZstdMtWindow, nextSrc) == 0);
assert!(std::mem::offset_of!(ZstdMtWindow, base) == size_of::<usize>());
assert!(std::mem::offset_of!(ZstdMtWindow, dictBase) == 2 * size_of::<usize>());
assert!(std::mem::offset_of!(ZstdMtWindow, dictLimit) == 3 * size_of::<usize>());
assert!(size_of::<ZstdMtWindow>() == if size_of::<usize>() == 8 { 40 } else { 24 });
};
/// ABI-compatible representation of `rawSeq` from `zstd_compress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZstdMtRawSeq {
pub offset: u32,
pub litLength: u32,
pub matchLength: u32,
}
/// ABI-compatible representation of `RawSeqStore_t` from
/// `zstd_compress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZstdMtRawSeqStore {
pub seq: *mut ZstdMtRawSeq,
pub pos: usize,
pub posInSequence: usize,
pub size: usize,
pub capacity: usize,
}
#[inline]
fn buffer_to_seq(buffer: ZstdMtBuffer) -> ZstdMtRawSeqStore {
ZstdMtRawSeqStore {
seq: buffer.start.cast::<ZstdMtRawSeq>(),
pos: 0,
posInSequence: 0,
size: 0,
capacity: buffer.capacity / mem::size_of::<ZstdMtRawSeq>(),
}
}
#[inline]
fn seq_to_buffer(seq: ZstdMtRawSeqStore) -> ZstdMtBuffer {
ZstdMtBuffer {
start: seq.seq.cast::<c_void>(),
capacity: seq.capacity.wrapping_mul(mem::size_of::<ZstdMtRawSeq>()),
}
}
#[inline]
fn is_overlapped(
buffer_start: *const c_void,
buffer_capacity: usize,
range_start: *const c_void,
range_size: usize,
) -> c_int {
if buffer_start.is_null() || range_start.is_null() {
return 0;
}
let buffer_start = buffer_start.cast::<u8>();
let range_start = range_start.cast::<u8>();
let buffer_end = buffer_start.wrapping_add(buffer_capacity);
let range_end = range_start.wrapping_add(range_size);
/* Empty ranges cannot overlap. */
if buffer_start == buffer_end || range_start == range_end {
return 0;
}
(buffer_start < range_end && range_start < buffer_end) as c_int
}
/// Convert a byte buffer into the raw-sequence store view used by the MT
/// sequence pool. The capacity is expressed in whole `rawSeq` elements, just
/// like the original C conversion leaf.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_bufferToSeq(buffer: ZstdMtBuffer) -> ZstdMtRawSeqStore {
buffer_to_seq(buffer)
}
/// Convert the raw-sequence store view back to a byte buffer for pool APIs.
/// The returned capacity is measured in bytes and follows C `size_t` wraparound
/// semantics for the multiplication.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_seqToBuffer(seq: ZstdMtRawSeqStore) -> ZstdMtBuffer {
seq_to_buffer(seq)
}
/// Return non-zero when two non-empty byte ranges overlap.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_isOverlapped(
bufferStart: *const c_void,
bufferCapacity: usize,
rangeStart: *const c_void,
rangeSize: usize,
) -> c_int {
is_overlapped(bufferStart, bufferCapacity, rangeStart, rangeSize)
}
/// Return non-zero when a buffer overlaps either the external dictionary or
/// the active prefix represented by a C `ZSTD_window_t`.
///
/// The C window stores the two ranges as pointer/index pairs. Passing those
/// scalar fields separately keeps the private C struct out of the Rust ABI;
/// the byte-distance arithmetic mirrors the original pointer subtraction.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_doesOverlapWindow(
bufferStart: *const c_void,
bufferCapacity: usize,
nextSrc: *const c_void,
base: *const c_void,
dictBase: *const c_void,
dictLimit: u32,
lowLimit: u32,
) -> c_int {
let ext_dict_start = dictBase.cast::<u8>().wrapping_add(lowLimit as usize);
let ext_dict_size = dictLimit.wrapping_sub(lowLimit) as usize;
let prefix_start = base.cast::<u8>().wrapping_add(dictLimit as usize);
let prefix_size = (nextSrc as usize)
.wrapping_sub(base as usize)
.wrapping_sub(dictLimit as usize);
(is_overlapped(
bufferStart,
bufferCapacity,
ext_dict_start.cast(),
ext_dict_size,
) != 0
|| is_overlapped(
bufferStart,
bufferCapacity,
prefix_start.cast(),
prefix_size,
) != 0) as c_int
}
/// Return non-zero when a buffer overlaps either range in a C window.
///
/// This keeps the original by-value internal ABI while the scalar helper
/// above remains available to projections that cannot expose the private
/// window layout.
#[no_mangle]
pub extern "C" fn ZSTDMT_doesOverlapWindow(buffer: ZstdMtBuffer, window: ZstdMtWindow) -> c_int {
ZSTDMT_rust_doesOverlapWindow(
buffer.start as *const c_void,
buffer.capacity,
window.nextSrc.cast(),
window.base.cast(),
window.dictBase.cast(),
window.dictLimit,
window.lowLimit,
)
}
#[derive(Default)]
struct BufferPoolState {
buffer_size: usize,
nb_buffers: usize,
}
/// The object is allocated with the caller's `ZSTD_customMem`; its mutex
/// protects all mutable pool state and the raw reusable-buffer array.
pub struct RustBufferPool {
custom_mem: ZstdCustomMem,
total_buffers: usize,
buffers: *mut MaybeUninit<ZstdMtBuffer>,
state: Mutex<BufferPoolState>,
}
// The raw array is only accessed while `state` is held. The pool itself is
// passed between C worker threads as an opaque pointer.
unsafe impl Send for RustBufferPool {}
unsafe impl Sync for RustBufferPool {}
/// The `ZSTD_CCtx *` values are created and destroyed by the existing C API;
/// Rust owns only the synchronized reusable-pointer pool.
pub struct RustCCtxPool {
custom_mem: ZstdCustomMem,
total_cctx: usize,
cctxs: *mut MaybeUninit<*mut c_void>,
state: Mutex<usize>,
}
unsafe impl Send for RustCCtxPool {}
unsafe impl Sync for RustCCtxPool {}
unsafe extern "C" {
fn ZSTD_createCCtx_advanced(custom_mem: ZstdCustomMem) -> *mut c_void;
fn ZSTD_freeCCtx(cctx: *mut c_void) -> usize;
fn ZSTD_sizeof_CCtx(cctx: *const c_void) -> usize;
}
unsafe fn custom_calloc(size: usize, custom_mem: ZstdCustomMem) -> *mut c_void {
if let Some(alloc) = custom_mem.customAlloc {
let allocation = unsafe { alloc(custom_mem.opaque, size) };
if !allocation.is_null() {
unsafe { ptr::write_bytes(allocation, 0, size) };
}
allocation
} else {
unsafe { libc::calloc(1, size) }
}
}
unsafe fn custom_malloc(size: usize, custom_mem: ZstdCustomMem) -> *mut c_void {
if let Some(alloc) = custom_mem.customAlloc {
unsafe { alloc(custom_mem.opaque, size) }
} else {
unsafe { libc::malloc(size) }
}
}
unsafe fn custom_free(allocation: *mut c_void, custom_mem: ZstdCustomMem) {
if allocation.is_null() {
return;
}
if let Some(free) = custom_mem.customFree {
unsafe { free(custom_mem.opaque, allocation) };
} else {
unsafe { libc::free(allocation) };
}
}
fn checked_array_size<T>(len: usize) -> Option<usize> {
mem::size_of::<T>().checked_mul(len)
}
fn rounded_job_count(requested: c_uint) -> Option<c_uint> {
if requested == 0 {
return None;
}
// The original C expression is `1 << (highbit(requested) + 1)`, which
// deliberately chooses a strictly larger power of two when requested is
// already a power of two.
let shift = usize::BITS - (requested as usize).leading_zeros();
let count = 1usize.checked_shl(shift)?;
if count > c_uint::MAX as usize {
return None;
}
Some(count as c_uint)
}
unsafe fn create_job_table(
nb_jobs_ptr: *mut c_uint,
job_size: usize,
custom_mem: ZstdCustomMem,
) -> *mut c_void {
if nb_jobs_ptr.is_null() || job_size == 0 {
return ptr::null_mut();
}
let Some(nb_jobs) = (unsafe { rounded_job_count(*nb_jobs_ptr) }) else {
return ptr::null_mut();
};
let Some(table_size) = job_size.checked_mul(nb_jobs as usize) else {
return ptr::null_mut();
};
let table = unsafe { custom_calloc(table_size, custom_mem) };
if table.is_null() {
return ptr::null_mut();
}
unsafe { *nb_jobs_ptr = nb_jobs };
table
}
unsafe fn free_job_table_storage(job_table: *mut c_void, custom_mem: ZstdCustomMem) {
if job_table.is_null() {
return;
}
unsafe { custom_free(job_table, custom_mem) };
}
type JobTableInitSync = unsafe extern "C" fn(*mut c_void, c_uint, usize) -> c_int;
type JobTableDestroySync = unsafe extern "C" fn(*mut c_void, c_uint, usize);
unsafe fn free_job_table_with_sync(
job_table: *mut c_void,
nb_jobs: c_uint,
job_size: usize,
custom_mem: ZstdCustomMem,
destroy_sync: JobTableDestroySync,
) {
if job_table.is_null() {
return;
}
unsafe { destroy_sync(job_table, nb_jobs, job_size) };
unsafe { ZSTDMT_rust_job_table_free(job_table, nb_jobs, job_size, custom_mem) };
}
unsafe fn create_job_table_with_sync(
nb_jobs_ptr: *mut c_uint,
job_size: usize,
custom_mem: ZstdCustomMem,
init_sync: JobTableInitSync,
destroy_sync: JobTableDestroySync,
) -> *mut c_void {
let job_table = unsafe { create_job_table(nb_jobs_ptr, job_size, custom_mem) };
if job_table.is_null() {
return ptr::null_mut();
}
let nb_jobs = unsafe { *nb_jobs_ptr };
if unsafe { init_sync(job_table, nb_jobs, job_size) } != 0 {
unsafe { free_job_table_with_sync(job_table, nb_jobs, job_size, custom_mem, destroy_sync) };
return ptr::null_mut();
}
job_table
}
unsafe fn expand_job_table(
job_table_ptr: *mut *mut c_void,
job_id_mask_ptr: *mut c_uint,
nb_workers: c_uint,
job_size: usize,
custom_mem: ZstdCustomMem,
init_sync: JobTableInitSync,
destroy_sync: JobTableDestroySync,
) -> usize {
if job_table_ptr.is_null() || job_id_mask_ptr.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let requested_jobs = nb_workers.wrapping_add(2);
let current_capacity = unsafe { (*job_id_mask_ptr).wrapping_add(1) };
if requested_jobs <= current_capacity {
return 0;
}
let old_job_table = unsafe { *job_table_ptr };
unsafe {
free_job_table_with_sync(
old_job_table,
current_capacity,
job_size,
custom_mem,
destroy_sync,
);
// Match ZSTDMT_expandJobsTable(): after the old table is released, a
// failed replacement leaves the C context with no table and a zero
// mask. The C wrapper copies these scalar results back to its private
// context only after this adapter returns.
*job_table_ptr = ptr::null_mut();
*job_id_mask_ptr = 0;
}
let mut nb_jobs = requested_jobs;
let new_job_table = unsafe {
create_job_table_with_sync(&mut nb_jobs, job_size, custom_mem, init_sync, destroy_sync)
};
if new_job_table.is_null() {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
debug_assert!(nb_jobs.is_power_of_two());
unsafe {
*job_table_ptr = new_job_table;
*job_id_mask_ptr = nb_jobs.wrapping_sub(1);
}
0
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_job_table_create(
nb_jobs_ptr: *mut c_uint,
job_size: usize,
custom_mem: ZstdCustomMem,
) -> *mut c_void {
unsafe { create_job_table(nb_jobs_ptr, job_size, custom_mem) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_createJobsTable(
nb_jobs_ptr: *mut c_uint,
job_size: usize,
custom_mem: ZstdCustomMem,
init_sync: JobTableInitSync,
destroy_sync: JobTableDestroySync,
) -> *mut c_void {
unsafe {
create_job_table_with_sync(nb_jobs_ptr, job_size, custom_mem, init_sync, destroy_sync)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_job_table_free(
job_table: *mut c_void,
_nb_jobs: c_uint,
_job_size: usize,
custom_mem: ZstdCustomMem,
) {
// The MT C adapter destroys its platform mutexes and condition variables
// before calling this storage-only release function. Keeping this Rust
// side free of MT-only C references also preserves single-threaded builds
// where zstdmt_compress.c is intentionally omitted.
unsafe { free_job_table_storage(job_table, custom_mem) }
}
/// Expand the C-owned MT job table when the requested worker count outgrows
/// its masked capacity. The table storage and custom allocator calls stay in
/// Rust; C supplies callbacks for the private descriptor synchronization
/// lifecycle.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_expandJobsTable(
job_table_ptr: *mut *mut c_void,
job_id_mask_ptr: *mut c_uint,
nb_workers: c_uint,
job_size: usize,
custom_mem: ZstdCustomMem,
init_sync: JobTableInitSync,
destroy_sync: JobTableDestroySync,
) -> usize {
unsafe {
expand_job_table(
job_table_ptr,
job_id_mask_ptr,
nb_workers,
job_size,
custom_mem,
init_sync,
destroy_sync,
)
}
}
unsafe fn create_buffer_pool(
max_nb_buffers: usize,
custom_mem: ZstdCustomMem,
) -> *mut RustBufferPool {
if max_nb_buffers == 0 {
return ptr::null_mut();
}
let Some(buffer_bytes) = checked_array_size::<MaybeUninit<ZstdMtBuffer>>(max_nb_buffers) else {
return ptr::null_mut();
};
let pool = unsafe { custom_calloc(mem::size_of::<RustBufferPool>(), custom_mem) }
.cast::<RustBufferPool>();
if pool.is_null() {
return ptr::null_mut();
}
let buffers =
unsafe { custom_calloc(buffer_bytes, custom_mem) }.cast::<MaybeUninit<ZstdMtBuffer>>();
if buffers.is_null() {
unsafe { custom_free(pool.cast(), custom_mem) };
return ptr::null_mut();
}
unsafe {
pool.write(RustBufferPool {
custom_mem,
total_buffers: max_nb_buffers,
buffers,
state: Mutex::new(BufferPoolState {
buffer_size: 64 << 10,
nb_buffers: 0,
}),
});
}
pool
}
unsafe fn destroy_buffer_pool(pool: *mut RustBufferPool) {
if pool.is_null() {
return;
}
let custom_mem = unsafe { (*pool).custom_mem };
let buffers = unsafe { (*pool).buffers };
let total_buffers = unsafe { (*pool).total_buffers };
for index in 0..total_buffers {
let buffer = unsafe { buffers.add(index).read().assume_init() };
unsafe { custom_free(buffer.start, custom_mem) };
}
unsafe { ptr::drop_in_place(pool) };
unsafe {
custom_free(buffers.cast(), custom_mem);
custom_free(pool.cast(), custom_mem);
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_create(
max_nb_buffers: c_uint,
custom_mem: ZstdCustomMem,
) -> *mut RustBufferPool {
unsafe { create_buffer_pool(max_nb_buffers as usize, custom_mem) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_free(pool: *mut RustBufferPool) {
unsafe { destroy_buffer_pool(pool) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_sizeof(pool: *const RustBufferPool) -> usize {
if pool.is_null() {
return 0;
}
let pool_ref = unsafe { &*pool };
let _state = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
let mut total_buffer_size = 0usize;
for index in 0..pool_ref.total_buffers {
let buffer = unsafe { pool_ref.buffers.add(index).read().assume_init() };
total_buffer_size = total_buffer_size.saturating_add(buffer.capacity);
}
mem::size_of::<RustBufferPool>()
.saturating_add(
pool_ref
.total_buffers
.saturating_mul(mem::size_of::<MaybeUninit<ZstdMtBuffer>>()),
)
.saturating_add(total_buffer_size)
}
#[inline]
fn sizeof_buffer_pool(wrapper_size: usize, rust_pool_size: usize) -> usize {
wrapper_size.wrapping_add(rust_pool_size)
}
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_sizeofBufferPool(wrapperSize: usize, rustPoolSize: usize) -> usize {
sizeof_buffer_pool(wrapperSize, rustPoolSize)
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_set_size(
pool: *mut RustBufferPool,
buffer_size: usize,
) {
if pool.is_null() {
return;
}
let pool_ref = unsafe { &*pool };
let mut state = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
state.buffer_size = buffer_size;
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_expand(
pool: *mut RustBufferPool,
max_nb_buffers: c_uint,
) -> *mut RustBufferPool {
if pool.is_null() {
return ptr::null_mut();
}
let max_nb_buffers = max_nb_buffers as usize;
let pool_ref = unsafe { &*pool };
let (total_buffers, buffer_size, custom_mem) = {
let state = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
(
pool_ref.total_buffers,
state.buffer_size,
pool_ref.custom_mem,
)
};
if total_buffers >= max_nb_buffers {
return pool;
}
// This matches the original resize contract: the old pool is consumed
// before creating the larger replacement.
unsafe { destroy_buffer_pool(pool) };
let replacement = unsafe { create_buffer_pool(max_nb_buffers, custom_mem) };
if !replacement.is_null() {
unsafe { ZSTDMT_rust_buffer_pool_set_size(replacement, buffer_size) };
}
replacement
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_get(pool: *mut RustBufferPool) -> ZstdMtBuffer {
if pool.is_null() {
return ZstdMtBuffer::default();
}
let pool_ref = unsafe { &*pool };
let (buffer_size, reusable) = {
let mut state = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
if state.buffer_size == 0 {
return ZstdMtBuffer::default();
}
if state.nb_buffers == 0 {
(state.buffer_size, ZstdMtBuffer::default())
} else {
state.nb_buffers -= 1;
let index = state.nb_buffers;
let buffer = unsafe { pool_ref.buffers.add(index).read().assume_init() };
unsafe {
pool_ref
.buffers
.add(index)
.write(MaybeUninit::new(ZstdMtBuffer::default()));
}
(state.buffer_size, buffer)
}
};
if !reusable.start.is_null()
&& reusable.capacity >= buffer_size
&& (reusable.capacity >> 3) <= buffer_size
{
return reusable;
}
if !reusable.start.is_null() {
unsafe { custom_free(reusable.start, pool_ref.custom_mem) };
}
let start = unsafe { custom_malloc(buffer_size, pool_ref.custom_mem) };
ZstdMtBuffer {
start,
capacity: if start.is_null() { 0 } else { buffer_size },
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_release(
pool: *mut RustBufferPool,
buffer: ZstdMtBuffer,
) {
if pool.is_null() || buffer.start.is_null() {
return;
}
let pool_ref = unsafe { &*pool };
let mut state = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
if state.nb_buffers < pool_ref.total_buffers {
let index = state.nb_buffers;
unsafe {
pool_ref.buffers.add(index).write(MaybeUninit::new(buffer));
}
state.nb_buffers += 1;
return;
}
drop(state);
unsafe { custom_free(buffer.start, pool_ref.custom_mem) };
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_resize(
pool: *mut RustBufferPool,
buffer: ZstdMtBuffer,
) -> ZstdMtBuffer {
if pool.is_null() || buffer.start.is_null() {
return buffer;
}
let pool_ref = unsafe { &*pool };
let buffer_size = {
let state = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
state.buffer_size
};
if buffer.capacity >= buffer_size {
return buffer;
}
let start = unsafe { custom_malloc(buffer_size, pool_ref.custom_mem) };
if start.is_null() {
return buffer;
}
unsafe {
ptr::copy_nonoverlapping(
buffer.start.cast::<u8>(),
start.cast::<u8>(),
buffer.capacity,
);
custom_free(buffer.start, pool_ref.custom_mem);
}
ZstdMtBuffer {
start,
capacity: buffer_size,
}
}
unsafe fn create_cctx_pool(nb_workers: usize, custom_mem: ZstdCustomMem) -> *mut RustCCtxPool {
if nb_workers == 0 {
return ptr::null_mut();
}
let Some(cctx_bytes) = checked_array_size::<MaybeUninit<*mut c_void>>(nb_workers) else {
return ptr::null_mut();
};
let pool =
unsafe { custom_calloc(mem::size_of::<RustCCtxPool>(), custom_mem) }.cast::<RustCCtxPool>();
if pool.is_null() {
return ptr::null_mut();
}
let cctxs = unsafe { custom_calloc(cctx_bytes, custom_mem) }.cast::<MaybeUninit<*mut c_void>>();
if cctxs.is_null() {
unsafe { custom_free(pool.cast(), custom_mem) };
return ptr::null_mut();
}
let first = unsafe { ZSTD_createCCtx_advanced(custom_mem) };
if first.is_null() {
unsafe {
custom_free(cctxs.cast(), custom_mem);
custom_free(pool.cast(), custom_mem);
}
return ptr::null_mut();
}
unsafe { cctxs.write(MaybeUninit::new(first)) };
unsafe {
pool.write(RustCCtxPool {
custom_mem,
total_cctx: nb_workers,
cctxs,
state: Mutex::new(1),
});
}
pool
}
unsafe fn destroy_cctx_pool(pool: *mut RustCCtxPool) {
if pool.is_null() {
return;
}
let custom_mem = unsafe { (*pool).custom_mem };
let cctxs = unsafe { (*pool).cctxs };
let total_cctx = unsafe { (*pool).total_cctx };
for index in 0..total_cctx {
let cctx = unsafe { cctxs.add(index).read().assume_init() };
unsafe { ZSTD_freeCCtx(cctx) };
}
unsafe { ptr::drop_in_place(pool) };
unsafe {
custom_free(cctxs.cast(), custom_mem);
custom_free(pool.cast(), custom_mem);
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_create(
nb_workers: c_uint,
custom_mem: ZstdCustomMem,
) -> *mut RustCCtxPool {
unsafe { create_cctx_pool(nb_workers as usize, custom_mem) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_free(pool: *mut RustCCtxPool) {
unsafe { destroy_cctx_pool(pool) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_sizeof(pool: *const RustCCtxPool) -> usize {
if pool.is_null() {
return 0;
}
let pool_ref = unsafe { &*pool };
let _state = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
let mut total_cctx_size = 0usize;
for index in 0..pool_ref.total_cctx {
let cctx = unsafe { pool_ref.cctxs.add(index).read().assume_init() };
total_cctx_size =
total_cctx_size.saturating_add(unsafe { ZSTD_sizeof_CCtx(cctx.cast_const()) });
}
mem::size_of::<RustCCtxPool>()
.saturating_add(
pool_ref
.total_cctx
.saturating_mul(mem::size_of::<MaybeUninit<*mut c_void>>()),
)
.saturating_add(total_cctx_size)
}
#[inline]
fn sizeof_cctx_pool(wrapper_size: usize, rust_pool_size: usize) -> usize {
wrapper_size.wrapping_add(rust_pool_size)
}
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_sizeofCCtxPool(wrapperSize: usize, rustPoolSize: usize) -> usize {
sizeof_cctx_pool(wrapperSize, rustPoolSize)
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_expand(
pool: *mut RustCCtxPool,
nb_workers: c_uint,
) -> *mut RustCCtxPool {
if pool.is_null() {
return ptr::null_mut();
}
let nb_workers = nb_workers as usize;
let pool_ref = unsafe { &*pool };
if pool_ref.total_cctx >= nb_workers {
return pool;
}
let custom_mem = pool_ref.custom_mem;
unsafe { destroy_cctx_pool(pool) };
unsafe { create_cctx_pool(nb_workers, custom_mem) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_get(pool: *mut RustCCtxPool) -> *mut c_void {
if pool.is_null() {
return ptr::null_mut();
}
let pool_ref = unsafe { &*pool };
let cctx = {
let mut available = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
if *available == 0 {
ptr::null_mut()
} else {
*available -= 1;
let index = *available;
unsafe { pool_ref.cctxs.add(index).read().assume_init() }
}
};
if !cctx.is_null() {
cctx
} else {
unsafe { ZSTD_createCCtx_advanced(pool_ref.custom_mem) }
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_release(pool: *mut RustCCtxPool, cctx: *mut c_void) {
if pool.is_null() || cctx.is_null() {
return;
}
let pool_ref = unsafe { &*pool };
let mut available = pool_ref
.state
.lock()
.unwrap_or_else(|error| error.into_inner());
if *available < pool_ref.total_cctx {
let index = *available;
unsafe {
pool_ref.cctxs.add(index).write(MaybeUninit::new(cctx));
}
*available += 1;
return;
}
drop(available);
unsafe { ZSTD_freeCCtx(cctx) };
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::{cell::RefCell, rc::Rc};
const DEFAULT_MEM: ZstdCustomMem = ZstdCustomMem {
customAlloc: None,
customFree: None,
opaque: ptr::null_mut(),
};
static JOB_TABLE_INIT_CALLS: AtomicUsize = AtomicUsize::new(0);
static JOB_TABLE_DESTROY_CALLS: AtomicUsize = AtomicUsize::new(0);
static JOB_TABLE_FAIL_INIT: AtomicBool = AtomicBool::new(false);
static JOB_TABLE_TEST_LOCK: Mutex<()> = Mutex::new(());
#[derive(Default)]
struct FreeCCtxTestContext {
events: Vec<&'static str>,
}
#[derive(Default)]
struct CreateCCtxTestContext {
events: Vec<&'static str>,
workers: c_uint,
jobs: c_uint,
factory_ok: bool,
resource_calls: usize,
}
fn record_create_cctx_event(context: *mut c_void, event: &'static str) {
unsafe {
(*context.cast::<CreateCCtxTestContext>())
.events
.push(event);
}
}
unsafe extern "C" fn create_cctx_test_allocate(
context: *mut c_void,
_size: usize,
) -> *mut c_void {
record_create_cctx_event(context, "allocate");
ptr::dangling_mut::<c_void>()
}
unsafe extern "C" fn create_cctx_test_set_workers(
context: *mut c_void,
workers: c_uint,
) -> usize {
record_create_cctx_event(context, "workers");
unsafe { (*context.cast::<CreateCCtxTestContext>()).workers = workers };
0
}
unsafe extern "C" fn create_cctx_test_initial_state(context: *mut c_void) {
record_create_cctx_event(context, "initial-state");
}
unsafe extern "C" fn create_cctx_test_factory(
context: *mut c_void,
_workers: c_uint,
) -> *mut c_void {
record_create_cctx_event(context, "factory");
let context = unsafe { &*context.cast::<CreateCCtxTestContext>() };
if context.factory_ok {
ptr::dangling_mut::<c_void>()
} else {
ptr::null_mut()
}
}
unsafe extern "C" fn create_cctx_test_jobs(
context: *mut c_void,
jobs: *mut c_uint,
) -> *mut c_void {
record_create_cctx_event(context, "jobs");
unsafe {
(*context.cast::<CreateCCtxTestContext>()).jobs = 8;
*jobs = 8;
}
ptr::dangling_mut::<c_void>()
}
unsafe extern "C" fn create_cctx_test_resource(
context: *mut c_void,
_workers: c_uint,
) -> *mut c_void {
let event = unsafe {
let state = &mut *context.cast::<CreateCCtxTestContext>();
let event = match state.resource_calls {
0 => "buffer-pool",
1 => "cctx-pool",
_ => "seq-pool",
};
state.resource_calls += 1;
event
};
record_create_cctx_event(context, event);
ptr::dangling_mut::<c_void>()
}
unsafe extern "C" fn create_cctx_test_serial_init(context: *mut c_void) -> c_int {
record_create_cctx_event(context, "serial");
0
}
unsafe extern "C" fn create_cctx_test_free(context: *mut c_void) {
record_create_cctx_event(context, "free");
}
fn create_cctx_test_projection(
context: &mut CreateCCtxTestContext,
custom_mem: ZstdCustomMem,
) -> ZSTDMT_RustCreateCCtxProjection {
ZSTDMT_RustCreateCCtxProjection {
callback_context: context as *mut CreateCCtxTestContext as *mut c_void,
requested_nb_workers: 99,
max_nb_workers: 4,
context_size: 128,
custom_mem,
}
}
#[derive(Default)]
struct ResizeTestContext {
events: Vec<&'static str>,
resource_calls: usize,
fail: Option<&'static str>,
}
fn record_resize_event(context: *mut c_void, event: &'static str) -> bool {
let state = unsafe { &mut *context.cast::<ResizeTestContext>() };
state.events.push(event);
state.fail == Some(event)
}
unsafe extern "C" fn resize_test_step(context: *mut c_void, _workers: c_uint) -> usize {
let state = unsafe { &*context.cast::<ResizeTestContext>() };
let event = match state.events.len() {
0 => "factory",
_ => "jobs",
};
if record_resize_event(context, event) {
ERROR(ZstdErrorCode::MemoryAllocation)
} else {
0
}
}
unsafe extern "C" fn resize_test_resource(
context: *mut c_void,
_workers: c_uint,
) -> *mut c_void {
let state = unsafe { &mut *context.cast::<ResizeTestContext>() };
let event = match state.resource_calls {
0 => "buffer-pool",
1 => "cctx-pool",
_ => "seq-pool",
};
state.resource_calls += 1;
if record_resize_event(context, event) {
ptr::null_mut()
} else {
ptr::dangling_mut::<c_void>()
}
}
unsafe extern "C" fn resize_test_set_workers(context: *mut c_void, _workers: c_uint) -> usize {
assert!(!record_resize_event(context, "set-workers"));
0
}
fn resize_test_projection(
context: &mut ResizeTestContext,
workers: c_uint,
) -> ZSTDMT_RustResizeProjection {
ZSTDMT_RustResizeProjection {
callback_context: context as *mut ResizeTestContext as *mut c_void,
nb_workers: workers,
}
}
#[test]
fn resize_preserves_resource_order_and_publishes_workers_last() {
let mut context = ResizeTestContext::default();
let projection = resize_test_projection(&mut context, 6);
let result = unsafe {
ZSTDMT_rust_resize(
&projection,
Some(resize_test_step),
Some(resize_test_step),
Some(resize_test_resource),
Some(resize_test_resource),
Some(resize_test_resource),
Some(resize_test_set_workers),
)
};
assert_eq!(result, 0);
assert_eq!(
context.events,
[
"factory",
"jobs",
"buffer-pool",
"cctx-pool",
"seq-pool",
"set-workers",
]
);
}
#[test]
fn resize_stops_after_resource_failure_without_publishing_workers() {
let mut context = ResizeTestContext {
fail: Some("cctx-pool"),
..ResizeTestContext::default()
};
let projection = resize_test_projection(&mut context, 3);
let result = unsafe {
ZSTDMT_rust_resize(
&projection,
Some(resize_test_step),
Some(resize_test_step),
Some(resize_test_resource),
Some(resize_test_resource),
Some(resize_test_resource),
Some(resize_test_set_workers),
)
};
assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation));
assert_eq!(
context.events,
["factory", "jobs", "buffer-pool", "cctx-pool"]
);
}
fn record_free_cctx_event(context: *mut c_void, event: &'static str) {
unsafe {
(*context.cast::<FreeCCtxTestContext>()).events.push(event);
}
}
unsafe extern "C" fn free_cctx_test_factory(context: *mut c_void) {
record_free_cctx_event(context, "factory");
}
unsafe extern "C" fn free_cctx_test_release_jobs(context: *mut c_void) {
record_free_cctx_event(context, "release-jobs");
}
unsafe extern "C" fn free_cctx_test_jobs(context: *mut c_void) {
record_free_cctx_event(context, "jobs");
}
unsafe extern "C" fn free_cctx_test_buffer_pool(context: *mut c_void) {
record_free_cctx_event(context, "buffer-pool");
}
unsafe extern "C" fn free_cctx_test_cctx_pool(context: *mut c_void) {
record_free_cctx_event(context, "cctx-pool");
}
unsafe extern "C" fn free_cctx_test_seq_pool(context: *mut c_void) {
record_free_cctx_event(context, "seq-pool");
}
unsafe extern "C" fn free_cctx_test_serial_state(context: *mut c_void) {
record_free_cctx_event(context, "serial-state");
}
unsafe extern "C" fn free_cctx_test_cdict(context: *mut c_void) {
record_free_cctx_event(context, "cdict");
}
unsafe extern "C" fn free_cctx_test_round_buffer(context: *mut c_void) {
record_free_cctx_event(context, "round-buffer");
}
unsafe extern "C" fn free_cctx_test_mtctx(context: *mut c_void) {
record_free_cctx_event(context, "mtctx");
}
fn free_cctx_test_state(
context: &mut FreeCCtxTestContext,
provided_factory: usize,
round_buffer_present: usize,
) -> ZSTDMT_RustFreeCCtxState {
ZSTDMT_RustFreeCCtxState {
callback_context: context as *mut FreeCCtxTestContext as *mut c_void,
provided_factory,
round_buffer_present,
free_factory: Some(free_cctx_test_factory),
release_all_job_resources: Some(free_cctx_test_release_jobs),
free_jobs: Some(free_cctx_test_jobs),
free_buffer_pool: Some(free_cctx_test_buffer_pool),
free_cctx_pool: Some(free_cctx_test_cctx_pool),
free_seq_pool: Some(free_cctx_test_seq_pool),
free_serial_state: Some(free_cctx_test_serial_state),
free_cdict: Some(free_cctx_test_cdict),
free_round_buffer: Some(free_cctx_test_round_buffer),
free_mtctx: Some(free_cctx_test_mtctx),
}
}
#[test]
fn free_cctx_null_state_returns_zero_without_callbacks() {
assert_eq!(unsafe { ZSTDMT_rust_freeCCtx(ptr::null()) }, 0);
}
#[test]
fn create_cctx_preserves_resource_order_and_clamps_workers() {
let mut context = CreateCCtxTestContext {
factory_ok: true,
..CreateCCtxTestContext::default()
};
let projection = create_cctx_test_projection(&mut context, DEFAULT_MEM);
let result = unsafe {
ZSTDMT_rust_createCCtx(
&projection,
Some(create_cctx_test_allocate),
Some(create_cctx_test_set_workers),
Some(create_cctx_test_initial_state),
Some(create_cctx_test_factory),
Some(create_cctx_test_jobs),
Some(create_cctx_test_resource),
Some(create_cctx_test_resource),
Some(create_cctx_test_resource),
Some(create_cctx_test_serial_init),
Some(create_cctx_test_free),
)
};
assert_eq!(result, ptr::dangling_mut::<c_void>());
assert_eq!(context.workers, 4);
assert_eq!(context.jobs, 8);
assert_eq!(
context.events,
vec![
"allocate",
"workers",
"initial-state",
"factory",
"jobs",
"buffer-pool",
"cctx-pool",
"seq-pool",
"serial",
]
);
}
#[test]
fn create_cctx_runs_remaining_callbacks_before_failure_cleanup() {
let mut context = CreateCCtxTestContext::default();
let projection = create_cctx_test_projection(&mut context, DEFAULT_MEM);
let result = unsafe {
ZSTDMT_rust_createCCtx(
&projection,
Some(create_cctx_test_allocate),
Some(create_cctx_test_set_workers),
Some(create_cctx_test_initial_state),
Some(create_cctx_test_factory),
Some(create_cctx_test_jobs),
Some(create_cctx_test_resource),
Some(create_cctx_test_resource),
Some(create_cctx_test_resource),
Some(create_cctx_test_serial_init),
Some(create_cctx_test_free),
)
};
assert!(result.is_null());
assert_eq!(
context.events,
vec![
"allocate",
"workers",
"initial-state",
"factory",
"jobs",
"buffer-pool",
"cctx-pool",
"seq-pool",
"serial",
"free",
]
);
}
unsafe extern "C" fn create_cctx_test_alloc(_opaque: *mut c_void, _size: usize) -> *mut c_void {
ptr::null_mut()
}
#[test]
fn create_cctx_rejects_mismatched_custom_allocator() {
let mut context = CreateCCtxTestContext {
factory_ok: true,
..CreateCCtxTestContext::default()
};
let invalid_mem = ZstdCustomMem {
customAlloc: Some(create_cctx_test_alloc),
customFree: None,
opaque: ptr::null_mut(),
};
let projection = create_cctx_test_projection(&mut context, invalid_mem);
let result = unsafe {
ZSTDMT_rust_createCCtx(
&projection,
Some(create_cctx_test_allocate),
Some(create_cctx_test_set_workers),
Some(create_cctx_test_initial_state),
Some(create_cctx_test_factory),
Some(create_cctx_test_jobs),
Some(create_cctx_test_resource),
Some(create_cctx_test_resource),
Some(create_cctx_test_resource),
Some(create_cctx_test_serial_init),
Some(create_cctx_test_free),
)
};
assert!(result.is_null());
assert!(context.events.is_empty());
}
#[test]
fn free_cctx_preserves_callback_order_and_ownership_conditions() {
let mut context = FreeCCtxTestContext::default();
let state = free_cctx_test_state(&mut context, 0, 1);
assert_eq!(unsafe { ZSTDMT_rust_freeCCtx(&state) }, 0);
assert_eq!(
context.events,
vec![
"factory",
"release-jobs",
"jobs",
"buffer-pool",
"cctx-pool",
"seq-pool",
"serial-state",
"cdict",
"round-buffer",
"mtctx",
]
);
context.events.clear();
let state = free_cctx_test_state(&mut context, 1, 0);
assert_eq!(unsafe { ZSTDMT_rust_freeCCtx(&state) }, 0);
assert_eq!(
context.events,
vec![
"release-jobs",
"jobs",
"buffer-pool",
"cctx-pool",
"seq-pool",
"serial-state",
"cdict",
"mtctx",
]
);
}
struct MockChunkCompressor {
continue_inputs: Vec<usize>,
end_inputs: Vec<usize>,
continue_results: Vec<usize>,
end_results: Vec<usize>,
}
#[derive(Default)]
struct MockChunkProgress {
calls: Vec<(usize, usize)>,
}
#[derive(Default)]
struct MockCompressionJob {
events: Vec<&'static str>,
errors: Vec<usize>,
finished: Vec<usize>,
consumed: Vec<usize>,
}
struct WaitForLdmTestContext {
events: Vec<&'static str>,
overlaps: Vec<bool>,
next_overlap: usize,
}
struct SerialWaitForTurnTestContext {
events: Vec<&'static str>,
next_job_id: c_uint,
wake_job_id: c_uint,
}
unsafe extern "C" fn serial_wait_for_turn_test_lock(context: *mut c_void) {
unsafe {
(*context.cast::<SerialWaitForTurnTestContext>())
.events
.push("lock")
};
}
unsafe extern "C" fn serial_wait_for_turn_test_wait(context: *mut c_void) {
unsafe {
let context = &mut *context.cast::<SerialWaitForTurnTestContext>();
context.events.push("wait");
context.next_job_id = context.wake_job_id;
}
}
fn serial_wait_for_turn_test_state(
context: &mut SerialWaitForTurnTestContext,
job_id: c_uint,
) -> ZSTDMT_RustSerialWaitForTurnState {
ZSTDMT_RustSerialWaitForTurnState {
callback_context: context as *mut _ as *mut c_void,
next_job_id: &mut context.next_job_id,
lock: Some(serial_wait_for_turn_test_lock),
wait: Some(serial_wait_for_turn_test_wait),
job_id,
}
}
struct SerialAdvanceTestContext {
events: Vec<&'static str>,
}
unsafe extern "C" fn serial_advance_test_broadcast(context: *mut c_void) {
unsafe {
(*context.cast::<SerialAdvanceTestContext>())
.events
.push("broadcast")
};
}
unsafe extern "C" fn serial_advance_test_unlock(context: *mut c_void) {
unsafe {
(*context.cast::<SerialAdvanceTestContext>())
.events
.push("unlock")
};
}
fn serial_advance_test_state(
context: &mut SerialAdvanceTestContext,
next_job_id: &mut c_uint,
) -> ZSTDMT_RustSerialAdvanceState {
ZSTDMT_RustSerialAdvanceState {
callback_context: context as *mut _ as *mut c_void,
next_job_id,
broadcast: Some(serial_advance_test_broadcast),
unlock: Some(serial_advance_test_unlock),
}
}
struct SerialEnsureFinishedTestContext {
events: Vec<&'static str>,
}
unsafe extern "C" fn serial_ensure_finished_test_lock(context: *mut c_void) {
unsafe {
(*context.cast::<SerialEnsureFinishedTestContext>())
.events
.push("lock")
};
}
unsafe extern "C" fn serial_ensure_finished_test_broadcast(context: *mut c_void) {
unsafe {
(*context.cast::<SerialEnsureFinishedTestContext>())
.events
.push("broadcast")
};
}
unsafe extern "C" fn serial_ensure_finished_test_ldm_lock(context: *mut c_void) {
unsafe {
(*context.cast::<SerialEnsureFinishedTestContext>())
.events
.push("ldm_lock")
};
}
unsafe extern "C" fn serial_ensure_finished_test_clear_ldm(context: *mut c_void) {
unsafe {
(*context.cast::<SerialEnsureFinishedTestContext>())
.events
.push("clear_ldm")
};
}
unsafe extern "C" fn serial_ensure_finished_test_ldm_signal(context: *mut c_void) {
unsafe {
(*context.cast::<SerialEnsureFinishedTestContext>())
.events
.push("ldm_signal")
};
}
unsafe extern "C" fn serial_ensure_finished_test_ldm_unlock(context: *mut c_void) {
unsafe {
(*context.cast::<SerialEnsureFinishedTestContext>())
.events
.push("ldm_unlock")
};
}
unsafe extern "C" fn serial_ensure_finished_test_unlock(context: *mut c_void) {
unsafe {
(*context.cast::<SerialEnsureFinishedTestContext>())
.events
.push("unlock")
};
}
unsafe extern "C" fn serial_ensure_finished_test_on_skip(
context: *mut c_void,
job_id: c_uint,
c_size: usize,
) {
assert_eq!(job_id, 4);
assert_eq!(c_size, 123);
unsafe {
(*context.cast::<SerialEnsureFinishedTestContext>())
.events
.push("skip")
};
}
fn serial_ensure_finished_test_state(
context: &mut SerialEnsureFinishedTestContext,
next_job_id: &mut c_uint,
job_id: c_uint,
c_size: usize,
) -> ZSTDMT_RustSerialEnsureFinishedState {
ZSTDMT_RustSerialEnsureFinishedState {
callback_context: context as *mut _ as *mut c_void,
next_job_id,
lock: Some(serial_ensure_finished_test_lock),
broadcast: Some(serial_ensure_finished_test_broadcast),
ldm_lock: Some(serial_ensure_finished_test_ldm_lock),
clear_ldm_window: Some(serial_ensure_finished_test_clear_ldm),
ldm_signal: Some(serial_ensure_finished_test_ldm_signal),
ldm_unlock: Some(serial_ensure_finished_test_ldm_unlock),
unlock: Some(serial_ensure_finished_test_unlock),
on_skip: Some(serial_ensure_finished_test_on_skip),
c_size,
job_id,
}
}
unsafe extern "C" fn wait_for_ldm_test_lock(context: *mut c_void) {
unsafe {
(*context.cast::<WaitForLdmTestContext>())
.events
.push("lock")
};
}
unsafe extern "C" fn wait_for_ldm_test_overlaps(
context: *mut c_void,
_buffer_start: *mut c_void,
_buffer_capacity: usize,
) -> c_int {
let context = unsafe { &mut *context.cast::<WaitForLdmTestContext>() };
context.events.push("overlap");
let overlaps = context
.overlaps
.get(context.next_overlap)
.copied()
.unwrap_or(false);
context.next_overlap += 1;
c_int::from(overlaps)
}
unsafe extern "C" fn wait_for_ldm_test_wait(context: *mut c_void) {
unsafe {
(*context.cast::<WaitForLdmTestContext>())
.events
.push("wait")
};
}
unsafe extern "C" fn wait_for_ldm_test_unlock(context: *mut c_void) {
unsafe {
(*context.cast::<WaitForLdmTestContext>())
.events
.push("unlock")
};
}
fn wait_for_ldm_test_state(
context: &mut WaitForLdmTestContext,
ldm_enabled: c_int,
) -> ZSTDMT_RustWaitForLdmState {
ZSTDMT_RustWaitForLdmState {
callback_context: context as *mut _ as *mut c_void,
buffer_start: ptr::null_mut(),
buffer_capacity: 8,
ldm_enabled,
lock: Some(wait_for_ldm_test_lock),
overlaps: Some(wait_for_ldm_test_overlaps),
wait: Some(wait_for_ldm_test_wait),
unlock: Some(wait_for_ldm_test_unlock),
}
}
#[test]
fn wait_for_ldm_runs_overlap_wait_loop_before_unlocking() {
let mut context = WaitForLdmTestContext {
events: Vec::new(),
overlaps: vec![true, false],
next_overlap: 0,
};
let state = wait_for_ldm_test_state(&mut context, ZSTD_PS_ENABLE);
unsafe { ZSTDMT_rust_waitForLdmComplete(&state) };
assert_eq!(
context.events,
vec!["lock", "overlap", "wait", "overlap", "unlock"]
);
assert_eq!(context.next_overlap, 2);
}
#[test]
fn wait_for_ldm_skips_callbacks_when_ldm_is_disabled() {
let mut context = WaitForLdmTestContext {
events: Vec::new(),
overlaps: vec![true],
next_overlap: 0,
};
let state = wait_for_ldm_test_state(&mut context, 0);
unsafe { ZSTDMT_rust_waitForLdmComplete(&state) };
assert!(context.events.is_empty());
assert_eq!(context.next_overlap, 0);
}
#[test]
fn serial_wait_for_turn_waits_until_job_is_ready() {
let mut context = SerialWaitForTurnTestContext {
events: Vec::new(),
next_job_id: 2,
wake_job_id: 4,
};
let state = serial_wait_for_turn_test_state(&mut context, 4);
let result = unsafe { ZSTDMT_rust_serialStateWaitForTurn(&state) };
assert_eq!(result, 1);
assert_eq!(context.events, vec!["lock", "wait"]);
assert_eq!(context.next_job_id, 4);
}
#[test]
fn serial_wait_for_turn_reports_skip_without_waiting_for_later_job() {
let mut context = SerialWaitForTurnTestContext {
events: Vec::new(),
next_job_id: 5,
wake_job_id: 5,
};
let state = serial_wait_for_turn_test_state(&mut context, 4);
let result = unsafe { ZSTDMT_rust_serialStateWaitForTurn(&state) };
assert_eq!(result, 0);
assert_eq!(context.events, vec!["lock"]);
assert_eq!(context.next_job_id, 5);
}
#[test]
fn serial_advance_increments_before_broadcast_and_unlock() {
let mut context = SerialAdvanceTestContext { events: Vec::new() };
let mut next_job_id = 7;
let state = serial_advance_test_state(&mut context, &mut next_job_id);
unsafe { ZSTDMT_rust_serialStateAdvance(&state) };
assert_eq!(next_job_id, 8);
assert_eq!(context.events, vec!["broadcast", "unlock"]);
}
#[test]
fn serial_advance_wraps_the_job_counter_like_c_unsigned_arithmetic() {
let mut context = SerialAdvanceTestContext { events: Vec::new() };
let mut next_job_id = c_uint::MAX;
let state = serial_advance_test_state(&mut context, &mut next_job_id);
unsafe { ZSTDMT_rust_serialStateAdvance(&state) };
assert_eq!(next_job_id, 0);
assert_eq!(context.events, vec!["broadcast", "unlock"]);
}
#[test]
fn serial_ensure_finished_runs_failed_job_cleanup_in_order() {
let mut context = SerialEnsureFinishedTestContext { events: Vec::new() };
let mut next_job_id = 4;
let state = serial_ensure_finished_test_state(&mut context, &mut next_job_id, 4, 123);
unsafe { ZSTDMT_rust_serialStateEnsureFinishedOrchestrated(&state) };
assert_eq!(next_job_id, 5);
assert_eq!(
context.events,
vec![
"lock",
"skip",
"broadcast",
"ldm_lock",
"clear_ldm",
"ldm_signal",
"ldm_unlock",
"unlock"
]
);
}
#[test]
fn serial_ensure_finished_unlocks_without_cleanup_for_later_job() {
let mut context = SerialEnsureFinishedTestContext { events: Vec::new() };
let mut next_job_id = 6;
let state = serial_ensure_finished_test_state(&mut context, &mut next_job_id, 4, 123);
unsafe { ZSTDMT_rust_serialStateEnsureFinishedOrchestrated(&state) };
assert_eq!(next_job_id, 6);
assert_eq!(context.events, vec!["lock", "unlock"]);
}
fn record_compression_job_event(state: &Rc<RefCell<MockCompressionJob>>, event: &'static str) {
state.borrow_mut().events.push(event);
}
fn record_compression_job_finish_event(context: *mut c_void, event: &'static str) {
unsafe {
(*context.cast::<RefCell<MockCompressionJob>>())
.borrow_mut()
.events
.push(event);
}
}
unsafe extern "C" fn compression_job_finish_test_ensure(context: *mut c_void) {
record_compression_job_finish_event(context, "ensure");
}
unsafe extern "C" fn compression_job_finish_test_error(context: *mut c_void, error: usize) {
unsafe {
let state = &*context.cast::<RefCell<MockCompressionJob>>();
state.borrow_mut().errors.push(error);
}
record_compression_job_finish_event(context, "error");
}
unsafe extern "C" fn compression_job_finish_test_last_block(
context: *mut c_void,
last_block_size: usize,
) {
unsafe {
let state = &*context.cast::<RefCell<MockCompressionJob>>();
state.borrow_mut().finished.push(last_block_size);
}
record_compression_job_finish_event(context, "last-block");
}
unsafe extern "C" fn compression_job_finish_test_release_seq(context: *mut c_void) {
record_compression_job_finish_event(context, "release-seq");
}
unsafe extern "C" fn compression_job_finish_test_release_cctx(context: *mut c_void) {
record_compression_job_finish_event(context, "release-cctx");
}
unsafe extern "C" fn compression_job_finish_test_consumed(
context: *mut c_void,
src_size: usize,
) {
unsafe {
let state = &*context.cast::<RefCell<MockCompressionJob>>();
state.borrow_mut().consumed.push(src_size);
}
record_compression_job_finish_event(context, "consumed");
}
unsafe extern "C" fn compression_job_finish_test_signal(context: *mut c_void) {
record_compression_job_finish_event(context, "signal");
}
fn compression_job_finish_test_projection(
state: &Rc<RefCell<MockCompressionJob>>,
src_size: usize,
) -> ZSTDMT_compressionJobFinishProjection {
ZSTDMT_compressionJobFinishProjection {
callbackContext: Rc::as_ptr(state).cast_mut().cast(),
srcSize: src_size,
ensureFinished: Some(compression_job_finish_test_ensure),
publishError: Some(compression_job_finish_test_error),
publishLastBlockSize: Some(compression_job_finish_test_last_block),
releaseSeq: Some(compression_job_finish_test_release_seq),
releaseCCtx: Some(compression_job_finish_test_release_cctx),
publishConsumed: Some(compression_job_finish_test_consumed),
signal: Some(compression_job_finish_test_signal),
}
}
#[test]
fn compression_job_frame_header_projection_writes_header_and_clears_repcodes() {
let mut stage = ZSTD_COMPRESSION_STAGE_INIT;
let mut rep_codes = [11u32, 22, 33];
let mut output = [0xa5u8; 18];
let state = ZSTDMT_compressionJobFrameHeaderState {
dst: output.as_mut_ptr().cast(),
dst_capacity: output.len(),
stage: &mut stage,
no_dict_id_flag: 0,
checksum_flag: 0,
content_size_flag: 1,
format: 0,
window_log: 20,
pledged_src_size_plus_one: 1,
dict_id: 0,
rep_codes: rep_codes.as_mut_ptr(),
};
let mut created_stage = ZSTD_COMPRESSION_STAGE_CREATED;
let created_state = ZSTDMT_compressionJobFrameHeaderState {
stage: &mut created_stage,
..state
};
let created = unsafe { ZSTDMT_rust_writeFrameHeader(&created_state) };
assert_eq!(created, ERROR(ZstdErrorCode::StageWrong));
assert_eq!(created_stage, ZSTD_COMPRESSION_STAGE_CREATED);
assert_eq!(rep_codes, [11, 22, 33]);
let result = unsafe { ZSTDMT_rust_writeFrameHeader(&state) };
assert_eq!(result, 6);
assert_eq!(stage, ZSTD_COMPRESSION_STAGE_ONGOING);
assert_eq!(rep_codes, [0; 3]);
assert_eq!(&output[..result], &[0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x00]);
assert_eq!(&output[result..], &[0xa5; 12]);
}
#[test]
fn compression_job_runs_first_job_stages_and_reports_final_block() {
let state = Rc::new(RefCell::new(MockCompressionJob::default()));
let acquire_state = Rc::clone(&state);
let prepare_state = Rc::clone(&state);
let sequence_state = Rc::clone(&state);
let begin_state = Rc::clone(&state);
let apply_state = Rc::clone(&state);
let compress_state = Rc::clone(&state);
let trace_state = Rc::clone(&state);
let finish = compression_job_finish_test_projection(&state, 19);
compression_job_with(
ZSTDMT_compressionJobProjection {
firstJob: 1,
lastJob: 1,
frameHeaderState: ptr::null(),
},
finish,
move || {
record_compression_job_event(&acquire_state, "acquire");
0
},
move || record_compression_job_event(&prepare_state, "prepare"),
move || record_compression_job_event(&sequence_state, "sequences"),
move || {
record_compression_job_event(&begin_state, "begin");
0
},
move || record_compression_job_event(&apply_state, "apply"),
|| panic!("first jobs do not rewrite a frame header"),
move |last_job| {
assert_eq!(last_job, 1);
record_compression_job_event(&compress_state, "compress");
ZSTDMT_chunkProcessResult {
error: 0,
lastBlockSize: 7,
}
},
move || record_compression_job_event(&trace_state, "trace"),
);
let state = state.borrow();
assert_eq!(
state.events,
vec![
"acquire",
"prepare",
"sequences",
"begin",
"apply",
"compress",
"trace",
"ensure",
"release-seq",
"release-cctx",
"last-block",
"consumed",
"signal",
]
);
assert!(state.errors.is_empty());
assert_eq!(state.finished, vec![7]);
assert_eq!(state.consumed, vec![19]);
}
#[test]
fn compression_job_normalizes_resource_error_before_finish_cleanup() {
let state = Rc::new(RefCell::new(MockCompressionJob::default()));
let acquire_state = Rc::clone(&state);
let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
let finish = compression_job_finish_test_projection(&state, 11);
compression_job_with(
ZSTDMT_compressionJobProjection {
firstJob: 1,
lastJob: 0,
frameHeaderState: ptr::null(),
},
finish,
move || {
record_compression_job_event(&acquire_state, "acquire");
expected_error
},
|| panic!("resource failure must skip parameter preparation"),
|| panic!("resource failure must skip sequence generation"),
|| panic!("resource failure must skip context initialization"),
|| panic!("resource failure must skip sequence application"),
|| panic!("resource failure must skip frame-header writing"),
|_| panic!("resource failure must skip compression"),
|| panic!("resource failure must skip tracing"),
);
let state = state.borrow();
assert_eq!(
state.events,
vec![
"acquire",
"error",
"ensure",
"release-seq",
"release-cctx",
"consumed",
"signal",
]
);
assert_eq!(state.errors, vec![expected_error]);
assert!(state.finished.is_empty());
assert_eq!(state.consumed, vec![11]);
}
#[test]
fn serial_turn_runs_ldm_before_checksum_and_advances_once() {
let events = Rc::new(RefCell::new(Vec::new()));
let wait_events = Rc::clone(&events);
let ldm_events = Rc::clone(&events);
let checksum_events = Rc::clone(&events);
let advance_events = Rc::clone(&events);
let mut seq_store = ZstdMtRawSeqStore::default();
serial_state_gen_sequences_with(
&mut seq_store,
ptr::null(),
8,
4,
true,
true,
move |job_id| {
assert_eq!(job_id, 4);
wait_events.borrow_mut().push("wait");
true
},
move |_seq_store, _src, src_size| {
assert_eq!(src_size, 8);
ldm_events.borrow_mut().push("ldm");
},
move |_src, src_size| {
assert_eq!(src_size, 8);
checksum_events.borrow_mut().push("checksum");
},
move || advance_events.borrow_mut().push("advance"),
);
assert_eq!(&*events.borrow(), &["wait", "ldm", "checksum", "advance"]);
}
#[test]
fn serial_turn_skips_failed_predecessor_and_keeps_empty_ldm_turn() {
let events = Rc::new(RefCell::new(Vec::new()));
let wait_events = Rc::clone(&events);
let advance_events = Rc::clone(&events);
let mut seq_store = ZstdMtRawSeqStore::default();
serial_state_gen_sequences_with(
&mut seq_store,
ptr::null(),
0,
9,
true,
true,
move |_job_id| {
wait_events.borrow_mut().push("wait");
false
},
|_seq_store, _src, _src_size| panic!("skipped jobs must not generate LDM sequences"),
|_src, _src_size| panic!("skipped jobs must not update the checksum"),
move || advance_events.borrow_mut().push("advance"),
);
assert_eq!(&*events.borrow(), &["wait", "advance"]);
let empty_events = Rc::new(RefCell::new(Vec::new()));
let empty_ldm_events = Rc::clone(&empty_events);
let empty_advance_events = Rc::clone(&empty_events);
serial_state_gen_sequences_with(
&mut seq_store,
ptr::null(),
0,
10,
true,
true,
|_job_id| true,
move |_seq_store, _src, src_size| {
assert_eq!(src_size, 0);
empty_ldm_events.borrow_mut().push("ldm");
},
|_src, _src_size| panic!("empty input must not update the checksum"),
move || empty_advance_events.borrow_mut().push("advance"),
);
assert_eq!(&*empty_events.borrow(), &["ldm", "advance"]);
}
#[test]
fn serial_state_ensure_finished_owns_skip_and_wrapping_policy() {
assert_eq!(
ZSTDMT_rust_serialStateEnsureFinished(4, 4),
ZSTDMT_serialStateEnsureFinishedResult {
skip: 1,
nextJobID: 5,
}
);
assert_eq!(
ZSTDMT_rust_serialStateEnsureFinished(3, 9),
ZSTDMT_serialStateEnsureFinishedResult {
skip: 1,
nextJobID: 10,
}
);
assert_eq!(
ZSTDMT_rust_serialStateEnsureFinished(11, 9),
ZSTDMT_serialStateEnsureFinishedResult {
skip: 0,
nextJobID: 11,
}
);
assert_eq!(
ZSTDMT_rust_serialStateEnsureFinished(c_uint::MAX, c_uint::MAX),
ZSTDMT_serialStateEnsureFinishedResult {
skip: 1,
nextJobID: 0,
}
);
}
struct WaitForAllJobsTestContext {
events: Vec<(c_uint, c_uint)>,
}
unsafe extern "C" fn wait_for_all_jobs_test_callback(
context: *mut c_void,
job_id: c_uint,
done_job_id: c_uint,
) {
unsafe {
(*context.cast::<WaitForAllJobsTestContext>())
.events
.push((job_id, done_job_id));
}
}
#[test]
fn wait_for_all_jobs_uses_ring_order_and_advances_to_next_job() {
let mut context = WaitForAllJobsTestContext { events: Vec::new() };
let done_job_id = unsafe {
ZSTDMT_rust_waitForAllJobsCompleted(
3,
6,
3,
(&mut context as *mut WaitForAllJobsTestContext).cast(),
Some(wait_for_all_jobs_test_callback),
)
};
assert_eq!(done_job_id, 6);
assert_eq!(context.events, vec![(3, 3), (0, 4), (1, 5)]);
}
struct ReleaseAllJobsTestContext {
job_ids: Vec<c_uint>,
}
unsafe extern "C" fn release_all_jobs_test_callback(context: *mut c_void, job_id: c_uint) {
unsafe {
(*context.cast::<ReleaseAllJobsTestContext>())
.job_ids
.push(job_id);
}
}
#[test]
fn release_all_jobs_visits_each_slot_in_order() {
let mut context = ReleaseAllJobsTestContext {
job_ids: Vec::new(),
};
unsafe {
ZSTDMT_rust_releaseAllJobResources(
3,
(&mut context as *mut ReleaseAllJobsTestContext).cast(),
Some(release_all_jobs_test_callback),
)
};
assert_eq!(context.job_ids, vec![0, 1, 2, 3]);
}
#[test]
fn compression_job_stops_on_non_first_chunk_error_and_cleans_up() {
let state = Rc::new(RefCell::new(MockCompressionJob::default()));
let acquire_state = Rc::clone(&state);
let prepare_state = Rc::clone(&state);
let sequence_state = Rc::clone(&state);
let begin_state = Rc::clone(&state);
let apply_state = Rc::clone(&state);
let header_state = Rc::clone(&state);
let compress_state = Rc::clone(&state);
let expected_error = ERROR(ZstdErrorCode::DstSizeTooSmall);
let finish = compression_job_finish_test_projection(&state, 23);
compression_job_with(
ZSTDMT_compressionJobProjection {
firstJob: 0,
lastJob: 1,
frameHeaderState: ptr::null(),
},
finish,
move || {
record_compression_job_event(&acquire_state, "acquire");
0
},
move || record_compression_job_event(&prepare_state, "prepare"),
move || record_compression_job_event(&sequence_state, "sequences"),
move || {
record_compression_job_event(&begin_state, "begin");
0
},
move || record_compression_job_event(&apply_state, "apply"),
move || {
record_compression_job_event(&header_state, "header");
0
},
move |last_job| {
assert_eq!(last_job, 1);
record_compression_job_event(&compress_state, "compress");
ZSTDMT_chunkProcessResult {
error: expected_error,
lastBlockSize: 9,
}
},
|| panic!("a failed chunk must not be traced"),
);
let state = state.borrow();
assert_eq!(
state.events,
vec![
"acquire",
"prepare",
"sequences",
"begin",
"apply",
"header",
"compress",
"error",
"ensure",
"release-seq",
"release-cctx",
"consumed",
"signal",
]
);
assert_eq!(state.errors, vec![expected_error]);
assert!(state.finished.is_empty());
assert_eq!(state.consumed, vec![23]);
}
fn init_projection() -> ZSTDMT_initCStreamProjection {
ZSTDMT_initCStreamProjection {
requestedNbWorkers: 4,
currentNbWorkers: 2,
jobSize: 1,
jobSizeMin: DEFAULT_ZSTDMT_JOBSIZE_MIN,
jobSizeMax: DEFAULT_ZSTDMT_JOBSIZE_MAX,
enableLdm: 0,
windowLog: 20,
chainLog: 16,
strategy: ZSTD_FAST,
overlapLog: 1,
rsyncable: 1,
roundBuffCapacity: 0,
allJobsCompleted: 0,
}
}
#[derive(Default)]
struct ResetStreamTestContext {
events: Vec<&'static str>,
}
unsafe extern "C" fn reset_stream_test_round_buffer(context: *mut c_void) {
unsafe {
(*context.cast::<ResetStreamTestContext>())
.events
.push("round");
}
}
unsafe extern "C" fn reset_stream_test_input_buffer(context: *mut c_void) {
unsafe {
(*context.cast::<ResetStreamTestContext>())
.events
.push("input");
}
}
unsafe extern "C" fn reset_stream_test_job_ids(context: *mut c_void) {
unsafe {
(*context.cast::<ResetStreamTestContext>())
.events
.push("jobs");
}
}
unsafe extern "C" fn reset_stream_test_frame_flags(context: *mut c_void) {
unsafe {
(*context.cast::<ResetStreamTestContext>())
.events
.push("flags");
}
}
unsafe extern "C" fn reset_stream_test_progress(context: *mut c_void) {
unsafe {
(*context.cast::<ResetStreamTestContext>())
.events
.push("progress");
}
}
fn reset_stream_test_state(context: *mut c_void) -> ZSTDMT_RustResetStreamState {
ZSTDMT_RustResetStreamState {
callback_context: context,
reset_round_buffer: Some(reset_stream_test_round_buffer),
reset_input_buffer: Some(reset_stream_test_input_buffer),
reset_job_ids: Some(reset_stream_test_job_ids),
reset_frame_flags: Some(reset_stream_test_frame_flags),
reset_progress: Some(reset_stream_test_progress),
}
}
#[test]
fn reset_stream_preserves_callback_order() {
let mut context = ResetStreamTestContext::default();
let state = reset_stream_test_state((&mut context as *mut ResetStreamTestContext).cast());
unsafe { ZSTDMT_rust_resetStream(&state) };
assert_eq!(
context.events,
vec!["round", "input", "jobs", "flags", "progress"]
);
}
#[test]
fn reset_stream_ignores_null_or_incomplete_state() {
let mut context = ResetStreamTestContext::default();
let state = ZSTDMT_RustResetStreamState {
callback_context: (&mut context as *mut ResetStreamTestContext).cast(),
reset_round_buffer: None,
reset_input_buffer: Some(reset_stream_test_input_buffer),
reset_job_ids: Some(reset_stream_test_job_ids),
reset_frame_flags: Some(reset_stream_test_frame_flags),
reset_progress: Some(reset_stream_test_progress),
};
unsafe {
ZSTDMT_rust_resetStream(&state);
ZSTDMT_rust_resetStream(ptr::null());
}
assert!(context.events.is_empty());
}
#[derive(Default)]
struct SerialResetTestContext {
events: Vec<&'static str>,
max_nb_seq: usize,
resize_result: c_int,
}
unsafe extern "C" fn serial_reset_test_next_job(context: *mut c_void) {
unsafe {
(*context.cast::<SerialResetTestContext>())
.events
.push("next-job");
}
}
unsafe extern "C" fn serial_reset_test_checksum(context: *mut c_void) {
unsafe {
(*context.cast::<SerialResetTestContext>())
.events
.push("checksum");
}
}
unsafe extern "C" fn serial_reset_test_set_nb_seq(context: *mut c_void, max_nb_seq: usize) {
unsafe {
let context = &mut *context.cast::<SerialResetTestContext>();
context.events.push("seq-size");
context.max_nb_seq = max_nb_seq;
}
}
unsafe extern "C" fn serial_reset_test_window(context: *mut c_void) {
unsafe {
(*context.cast::<SerialResetTestContext>())
.events
.push("window");
}
}
unsafe extern "C" fn serial_reset_test_resize(context: *mut c_void) -> c_int {
unsafe {
let context = &mut *context.cast::<SerialResetTestContext>();
context.events.push("resize");
context.resize_result
}
}
unsafe extern "C" fn serial_reset_test_zero(context: *mut c_void) {
unsafe {
(*context.cast::<SerialResetTestContext>())
.events
.push("zero");
}
}
unsafe extern "C" fn serial_reset_test_dictionary(context: *mut c_void) {
unsafe {
(*context.cast::<SerialResetTestContext>())
.events
.push("dictionary");
}
}
unsafe extern "C" fn serial_reset_test_copy_window(context: *mut c_void) {
unsafe {
(*context.cast::<SerialResetTestContext>())
.events
.push("copy-window");
}
}
type SerialResetTestCallbacks = (
Option<ZSTDMT_serialResetVoidFn>,
Option<ZSTDMT_serialResetVoidFn>,
Option<ZSTDMT_serialResetSetNbSeqFn>,
Option<ZSTDMT_serialResetVoidFn>,
Option<ZSTDMT_serialResetResizeFn>,
Option<ZSTDMT_serialResetVoidFn>,
Option<ZSTDMT_serialResetVoidFn>,
Option<ZSTDMT_serialResetVoidFn>,
);
fn serial_reset_test_callbacks() -> SerialResetTestCallbacks {
(
Some(serial_reset_test_next_job),
Some(serial_reset_test_checksum),
Some(serial_reset_test_set_nb_seq),
Some(serial_reset_test_window),
Some(serial_reset_test_resize),
Some(serial_reset_test_zero),
Some(serial_reset_test_dictionary),
Some(serial_reset_test_copy_window),
)
}
#[test]
fn serial_state_reset_runs_ldm_operations_in_c_order() {
let mut context = SerialResetTestContext::default();
let callbacks = serial_reset_test_callbacks();
let result = unsafe {
ZSTDMT_rust_serialStateReset(
ZSTD_PS_ENABLE,
1,
123,
(&mut context as *mut SerialResetTestContext).cast(),
callbacks.0,
callbacks.1,
callbacks.2,
callbacks.3,
callbacks.4,
callbacks.5,
callbacks.6,
callbacks.7,
)
};
assert_eq!(result, 0);
assert_eq!(context.max_nb_seq, 123);
assert_eq!(
context.events,
vec![
"next-job",
"checksum",
"seq-size",
"window",
"resize",
"zero",
"dictionary",
"copy-window",
]
);
}
#[test]
fn serial_state_reset_stops_before_zeroing_after_resize_failure() {
let mut context = SerialResetTestContext {
resize_result: 1,
..Default::default()
};
let callbacks = serial_reset_test_callbacks();
let result = unsafe {
ZSTDMT_rust_serialStateReset(
ZSTD_PS_ENABLE,
0,
123,
(&mut context as *mut SerialResetTestContext).cast(),
callbacks.0,
callbacks.1,
callbacks.2,
callbacks.3,
callbacks.4,
callbacks.5,
callbacks.6,
callbacks.7,
)
};
assert_eq!(result, 1);
assert_eq!(
context.events,
vec!["next-job", "seq-size", "window", "resize"]
);
}
#[derive(Default)]
struct SerialLifecycleTestContext {
events: Vec<&'static str>,
init_errors: [c_int; 4],
}
unsafe fn serial_lifecycle_init_step(
context: *mut c_void,
index: usize,
event: &'static str,
) -> c_int {
let context = unsafe { &mut *context.cast::<SerialLifecycleTestContext>() };
context.events.push(event);
context.init_errors[index]
}
unsafe extern "C" fn serial_lifecycle_zero(context: *mut c_void) {
unsafe {
(*context.cast::<SerialLifecycleTestContext>())
.events
.push("zero");
}
}
unsafe extern "C" fn serial_lifecycle_init_mutex(context: *mut c_void) -> c_int {
unsafe { serial_lifecycle_init_step(context, 0, "mutex") }
}
unsafe extern "C" fn serial_lifecycle_init_cond(context: *mut c_void) -> c_int {
unsafe { serial_lifecycle_init_step(context, 1, "cond") }
}
unsafe extern "C" fn serial_lifecycle_init_ldm_mutex(context: *mut c_void) -> c_int {
unsafe { serial_lifecycle_init_step(context, 2, "ldm-mutex") }
}
unsafe extern "C" fn serial_lifecycle_init_ldm_cond(context: *mut c_void) -> c_int {
unsafe { serial_lifecycle_init_step(context, 3, "ldm-cond") }
}
unsafe extern "C" fn serial_lifecycle_destroy_mutex(context: *mut c_void) {
unsafe {
(*context.cast::<SerialLifecycleTestContext>())
.events
.push("destroy-mutex");
}
}
unsafe extern "C" fn serial_lifecycle_destroy_cond(context: *mut c_void) {
unsafe {
(*context.cast::<SerialLifecycleTestContext>())
.events
.push("destroy-cond");
}
}
unsafe extern "C" fn serial_lifecycle_destroy_ldm_mutex(context: *mut c_void) {
unsafe {
(*context.cast::<SerialLifecycleTestContext>())
.events
.push("destroy-ldm-mutex");
}
}
unsafe extern "C" fn serial_lifecycle_destroy_ldm_cond(context: *mut c_void) {
unsafe {
(*context.cast::<SerialLifecycleTestContext>())
.events
.push("destroy-ldm-cond");
}
}
unsafe extern "C" fn serial_lifecycle_free_tables(context: *mut c_void) {
unsafe {
(*context.cast::<SerialLifecycleTestContext>())
.events
.push("free-tables");
}
}
#[test]
fn serial_state_lifecycle_preserves_order_and_aggregates_init_errors() {
let mut context = SerialLifecycleTestContext {
init_errors: [0, 2, 0, 4],
..Default::default()
};
let opaque = (&mut context as *mut SerialLifecycleTestContext).cast();
let result = unsafe {
ZSTDMT_rust_serialStateInit(
opaque,
Some(serial_lifecycle_zero),
Some(serial_lifecycle_init_mutex),
Some(serial_lifecycle_init_cond),
Some(serial_lifecycle_init_ldm_mutex),
Some(serial_lifecycle_init_ldm_cond),
)
};
assert_eq!(result, 6);
unsafe {
ZSTDMT_rust_serialStateFree(
opaque,
Some(serial_lifecycle_destroy_mutex),
Some(serial_lifecycle_destroy_cond),
Some(serial_lifecycle_destroy_ldm_mutex),
Some(serial_lifecycle_destroy_ldm_cond),
Some(serial_lifecycle_free_tables),
)
};
assert_eq!(
context.events,
vec![
"zero",
"mutex",
"cond",
"ldm-mutex",
"ldm-cond",
"destroy-mutex",
"destroy-cond",
"destroy-ldm-mutex",
"destroy-ldm-cond",
"free-tables",
]
);
}
#[test]
fn init_c_stream_preserves_success_order_and_normalization() {
let projection = init_projection();
let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
let values = Rc::new(RefCell::new(Vec::<(&'static str, usize)>::new()));
let resize_events = Rc::clone(&events);
let drain_events = Rc::clone(&events);
let apply_events = Rc::clone(&events);
let apply_values = Rc::clone(&values);
let prepare_events = Rc::clone(&events);
let prefix_events = Rc::clone(&events);
let prefix_values = Rc::clone(&values);
let section_events = Rc::clone(&events);
let section_values = Rc::clone(&values);
let rsync_events = Rc::clone(&events);
let buffer_events = Rc::clone(&events);
let buffer_values = Rc::clone(&values);
let round_events = Rc::clone(&events);
let round_values = Rc::clone(&values);
let reset_events = Rc::clone(&events);
let update_events = Rc::clone(&events);
let serial_events = Rc::clone(&events);
let serial_values = Rc::clone(&values);
let result = init_c_stream_with(
projection,
move |workers| {
assert_eq!(workers, 4);
resize_events.borrow_mut().push("resize");
0
},
move || drain_events.borrow_mut().push("drain"),
move |job_size| {
apply_events.borrow_mut().push("apply");
apply_values.borrow_mut().push(("job", job_size));
},
move || {
prepare_events.borrow_mut().push("prepare-dict");
0
},
move |size| {
prefix_events.borrow_mut().push("prefix");
prefix_values.borrow_mut().push(("prefix", size));
},
move |size| {
section_events.borrow_mut().push("section");
section_values.borrow_mut().push(("section", size));
},
move |_hit_mask, _prime_power| rsync_events.borrow_mut().push("rsync"),
move |size| {
buffer_events.borrow_mut().push("buffer");
buffer_values.borrow_mut().push(("buffer", size));
},
move |capacity| {
round_events.borrow_mut().push("round");
round_values.borrow_mut().push(("round", capacity));
0
},
move || reset_events.borrow_mut().push("reset"),
move || {
update_events.borrow_mut().push("update-dict");
0
},
move |size| {
serial_events.borrow_mut().push("serial");
serial_values.borrow_mut().push(("serial", size));
0
},
);
assert_eq!(result, 0);
assert_eq!(
events.borrow().as_slice(),
&[
"resize",
"drain",
"apply",
"prepare-dict",
"prefix",
"section",
"rsync",
"buffer",
"round",
"reset",
"update-dict",
"serial",
]
);
assert_eq!(values.borrow()[0], ("job", DEFAULT_ZSTDMT_JOBSIZE_MIN));
assert_eq!(values.borrow()[1], ("prefix", 0));
assert_eq!(values.borrow()[2], ("section", DEFAULT_ZSTDMT_JOBSIZE_MIN));
assert_eq!(values.borrow()[5], ("serial", DEFAULT_ZSTDMT_JOBSIZE_MIN));
assert!(values.borrow()[3].1 > 0);
assert!(values.borrow()[4].1 > 0);
}
#[test]
fn init_c_stream_stops_before_normalization_when_resize_fails() {
let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
let result = init_c_stream_with(
init_projection(),
|_| expected_error,
|| panic!("a failed resize must stop initialization"),
|_| panic!("a failed resize must stop initialization"),
|| panic!("a failed resize must stop initialization"),
|_| panic!("a failed resize must stop initialization"),
|_| panic!("a failed resize must stop initialization"),
|_, _| panic!("a failed resize must stop initialization"),
|_| panic!("a failed resize must stop initialization"),
|_| panic!("a failed resize must stop initialization"),
|| panic!("a failed resize must stop initialization"),
|| panic!("a failed resize must stop initialization"),
|_| panic!("a failed resize must stop initialization"),
);
assert_eq!(result, expected_error);
}
#[test]
fn init_c_stream_stops_after_round_buffer_error() {
let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
let mut projection = init_projection();
projection.currentNbWorkers = projection.requestedNbWorkers;
projection.allJobsCompleted = 1;
let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
let result = init_c_stream_with(
projection,
|_| panic!("workers already match"),
|| panic!("all jobs are complete"),
{
let events = Rc::clone(&events);
move |_| events.borrow_mut().push("apply")
},
{
let events = Rc::clone(&events);
move || {
events.borrow_mut().push("prepare");
0
}
},
{
let events = Rc::clone(&events);
move |_| events.borrow_mut().push("prefix")
},
{
let events = Rc::clone(&events);
move |_| events.borrow_mut().push("section")
},
{
let events = Rc::clone(&events);
move |_, _| events.borrow_mut().push("rsync")
},
{
let events = Rc::clone(&events);
move |_| events.borrow_mut().push("buffer")
},
{
let events = Rc::clone(&events);
move |_| {
events.borrow_mut().push("round");
expected_error
}
},
{
let events = Rc::clone(&events);
move || events.borrow_mut().push("reset")
},
{
let events = Rc::clone(&events);
move || {
events.borrow_mut().push("update");
0
}
},
|_| panic!("serial reset must follow successful dictionary update"),
);
assert_eq!(result, expected_error);
assert_eq!(
events.borrow().as_slice(),
&["apply", "prepare", "prefix", "section", "rsync", "buffer", "round"]
);
}
unsafe extern "C" fn mock_compress_continue(
cctx: *mut c_void,
_dst: *mut c_void,
_dst_capacity: usize,
_src: *const c_void,
src_size: usize,
) -> usize {
let compressor = unsafe { &mut *cctx.cast::<MockChunkCompressor>() };
let call_number = compressor.continue_inputs.len();
compressor.continue_inputs.push(src_size);
compressor
.continue_results
.get(call_number)
.copied()
.unwrap_or(0)
}
unsafe extern "C" fn mock_compress_end(
cctx: *mut c_void,
_dst: *mut c_void,
_dst_capacity: usize,
_src: *const c_void,
src_size: usize,
) -> usize {
let compressor = unsafe { &mut *cctx.cast::<MockChunkCompressor>() };
let call_number = compressor.end_inputs.len();
compressor.end_inputs.push(src_size);
compressor
.end_results
.get(call_number)
.copied()
.unwrap_or(0)
}
unsafe extern "C" fn mock_chunk_progress(context: *mut c_void, c_size: usize, consumed: usize) {
let progress = unsafe { &mut *context.cast::<MockChunkProgress>() };
progress.calls.push((c_size, consumed));
}
fn run_mock_chunk_loop(
compressor: &mut MockChunkCompressor,
progress: &mut MockChunkProgress,
src_size: usize,
chunk_size: usize,
last_job: c_uint,
) -> ZSTDMT_chunkProcessResult {
let src = [0u8; 32];
let mut dst = [0u8; 64];
unsafe {
compress_job_chunks_with(
(compressor as *mut MockChunkCompressor).cast(),
src.as_ptr().cast(),
src_size,
dst.as_mut_ptr().cast(),
dst.len(),
chunk_size,
last_job,
(progress as *mut MockChunkProgress).cast(),
Some(mock_chunk_progress),
mock_compress_continue,
mock_compress_end,
)
}
}
fn run_output_publication(
output: &mut [u8],
output_size: usize,
output_pos: usize,
job: &[u8],
job_capacity: usize,
c_size: usize,
dst_flushed: usize,
) -> ZSTDMT_flushPublicationResult {
unsafe {
publish_job_output_with(
output.as_mut_ptr().cast(),
output_size,
output_pos,
job.as_ptr().cast(),
job_capacity,
c_size,
dst_flushed,
)
}
}
#[test]
fn output_publication_handles_no_output_space() {
let mut output = [0xa5; 8];
let original = output;
let job = [1, 2, 3, 4];
let output_size = output.len();
let result = run_output_publication(
&mut output,
output_size,
output_size,
&job,
job.len(),
job.len(),
0,
);
assert_eq!(result.status, FLUSH_PUBLICATION_OK);
assert_eq!(result.toFlush, 0);
assert_eq!(result.outputPos, output.len());
assert_eq!(result.dstFlushed, 0);
assert_eq!(output, original);
}
#[test]
fn output_publication_handles_partial_flush_with_offsets() {
let mut output = [0xa5; 10];
let job = [0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17];
let result = run_output_publication(&mut output, 9, 2, &job, job.len(), 6, 1);
assert_eq!(result.status, FLUSH_PUBLICATION_OK);
assert_eq!(result.toFlush, 5);
assert_eq!(result.outputPos, 7);
assert_eq!(result.dstFlushed, 6);
assert_eq!(&output[..2], &[0xa5, 0xa5]);
assert_eq!(&output[2..7], &job[1..6]);
assert_eq!(&output[7..], &[0xa5, 0xa5, 0xa5]);
}
#[test]
fn output_publication_handles_complete_flush() {
let mut output = [0xa5; 8];
let job = [0x20, 0x21, 0x22, 0x23, 0x24, 0x25];
let output_size = output.len();
let result = run_output_publication(&mut output, output_size, 1, &job, job.len(), 6, 2);
assert_eq!(result.status, FLUSH_PUBLICATION_OK);
assert_eq!(result.toFlush, 4);
assert_eq!(result.outputPos, 5);
assert_eq!(result.dstFlushed, 6);
assert_eq!(&output[1..5], &job[2..6]);
assert_eq!(&output[..1], &[0xa5]);
assert_eq!(&output[5..], &[0xa5, 0xa5, 0xa5]);
}
#[test]
fn output_publication_handles_repeated_partial_flushes() {
let mut output = [0xa5; 8];
let job = [0x30, 0x31, 0x32, 0x33, 0x34, 0x35];
let first = run_output_publication(&mut output, 3, 0, &job, job.len(), 6, 0);
let second = run_output_publication(
&mut output,
6,
first.outputPos,
&job,
job.len(),
6,
first.dstFlushed,
);
assert_eq!(first.status, FLUSH_PUBLICATION_OK);
assert_eq!(first.toFlush, 3);
assert_eq!(first.outputPos, 3);
assert_eq!(first.dstFlushed, 3);
assert_eq!(second.status, FLUSH_PUBLICATION_OK);
assert_eq!(second.toFlush, 3);
assert_eq!(second.outputPos, 6);
assert_eq!(second.dstFlushed, 6);
assert_eq!(&output[..6], &job);
assert_eq!(&output[6..], &[0xa5, 0xa5]);
}
#[test]
fn output_publication_does_not_copy_trailing_sentinels() {
let mut output = [0xa5; 12];
let job = [0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0xde, 0xad];
let output_size = output.len();
let result = run_output_publication(&mut output, output_size, 4, &job, job.len(), 5, 2);
assert_eq!(result.status, FLUSH_PUBLICATION_OK);
assert_eq!(result.toFlush, 3);
assert_eq!(&output[..4], &[0xa5, 0xa5, 0xa5, 0xa5]);
assert_eq!(&output[4..7], &job[2..5]);
assert_eq!(&output[7..], &[0xa5, 0xa5, 0xa5, 0xa5, 0xa5]);
}
#[test]
fn output_publication_rejects_invalid_bounds_without_copy() {
let mut output = [0xa5; 8];
let original = output;
let job = [0x50, 0x51, 0x52, 0x53];
let result = run_output_publication(&mut output, 4, 5, &job, job.len(), 4, 0);
assert_eq!(result.status, FLUSH_PUBLICATION_INVALID_BOUNDS);
assert_eq!(result.toFlush, 0);
assert_eq!(result.outputPos, 5);
assert_eq!(result.dstFlushed, 0);
assert_eq!(output, original);
}
#[test]
fn create_job_rejects_full_table_without_callbacks() {
let projection = ZSTDMT_createJobProjection {
doneJobID: 0,
nextJobID: 8,
jobIDMask: 7,
..ZSTDMT_createJobProjection::default()
};
let result = create_compression_job_with(
projection,
|_job_id, _initialization| panic!("table-full job must not be prepared"),
|_job_id| panic!("table-full job must not be written"),
|_job_id| panic!("table-full job must not be posted"),
);
assert_eq!(result.action, CREATE_JOB_TABLE_FULL);
assert_eq!(result.jobID, 0);
assert_eq!(result.jobNumber, 8);
assert_eq!(result.nextJobID, 8);
assert_eq!(result.jobReady, 0);
}
#[test]
fn create_job_projects_ordinary_initialization_and_prefix_advance() {
let source = [0u8; 8];
let prefix = [1u8; 3];
let projection = ZSTDMT_createJobProjection {
doneJobID: 0,
nextJobID: 3,
jobIDMask: 7,
srcStart: source.as_ptr().cast(),
srcSize: source.len(),
inBuffFilled: source.len(),
prefixStart: prefix.as_ptr().cast(),
prefixSize: prefix.len(),
targetPrefixSize: 4,
checksumFlag: 1,
..ZSTDMT_createJobProjection::default()
};
let mut initialization = None;
let mut posted = None;
let result = create_compression_job_with(
projection,
|job_id, value| {
assert_eq!(job_id, 3);
initialization = Some(*value);
},
|_job_id| panic!("ordinary job must not use the empty-block path"),
|job_id| {
posted = Some(job_id);
true
},
);
let initialization = initialization.expect("job should be initialized");
assert_eq!(initialization.srcStart, source.as_ptr().cast());
assert_eq!(initialization.srcSize, source.len());
assert_eq!(initialization.prefixStart, prefix.as_ptr().cast());
assert_eq!(initialization.prefixSize, prefix.len());
assert_eq!(
initialization.nextPrefixStart,
source.as_ptr().wrapping_add(4).cast()
);
assert_eq!(initialization.nextPrefixSize, 4);
assert_eq!(initialization.roundBuffPosDelta, source.len());
assert_eq!(initialization.jobNumber, 3);
assert_eq!(initialization.firstJob, 0);
assert_eq!(initialization.lastJob, 0);
assert_eq!(initialization.frameChecksumNeeded, 0);
assert_eq!(posted, Some(3));
assert_eq!(result.action, CREATE_JOB_POST);
assert_eq!(result.jobID, 3);
assert_eq!(result.nextJobID, 4);
assert_eq!(result.jobReady, 0);
}
#[test]
fn create_job_projects_terminal_empty_frame() {
let prefix = [2u8; 2];
let projection = ZSTDMT_createJobProjection {
doneJobID: 0,
nextJobID: 2,
jobIDMask: 7,
inBuffFilled: 0,
prefixStart: prefix.as_ptr().cast(),
prefixSize: prefix.len(),
endFrame: 1,
checksumFlag: 1,
..ZSTDMT_createJobProjection::default()
};
let mut initialization = None;
let mut empty_job = None;
let result = create_compression_job_with(
projection,
|job_id, value| {
assert_eq!(job_id, 2);
initialization = Some(*value);
},
|job_id| empty_job = Some(job_id),
|_job_id| panic!("terminal empty job must not be posted"),
);
let initialization = initialization.expect("empty job should be initialized");
assert_eq!(initialization.srcSize, 0);
assert_eq!(initialization.prefixStart, prefix.as_ptr().cast());
assert_eq!(initialization.prefixSize, prefix.len());
assert!(initialization.nextPrefixStart.is_null());
assert_eq!(initialization.nextPrefixSize, 0);
assert_eq!(initialization.roundBuffPosDelta, 0);
assert_eq!(initialization.jobNumber, 2);
assert_eq!(initialization.firstJob, 0);
assert_eq!(initialization.lastJob, 1);
assert_eq!(initialization.frameChecksumNeeded, 1);
assert_eq!(initialization.clearChecksumFlag, 0);
assert_eq!(empty_job, Some(2));
assert_eq!(result.action, CREATE_JOB_EMPTY);
assert_eq!(result.nextJobID, 3);
assert_eq!(result.jobReady, 0);
}
#[test]
fn create_job_preserves_pool_success_and_failure_state() {
let projection = ZSTDMT_createJobProjection {
doneJobID: 0,
nextJobID: 4,
jobIDMask: 7,
jobReady: 1,
..ZSTDMT_createJobProjection::default()
};
let success = create_compression_job_with(
projection,
|_job_id, _initialization| panic!("prepared job must not be reinitialized"),
|_job_id| panic!("prepared job must not use the empty-block path"),
|job_id| {
assert_eq!(job_id, 4);
true
},
);
assert_eq!(success.action, CREATE_JOB_POST);
assert_eq!(success.nextJobID, 5);
assert_eq!(success.jobReady, 0);
let failure = create_compression_job_with(
projection,
|_job_id, _initialization| panic!("prepared job must not be reinitialized"),
|_job_id| panic!("prepared job must not use the empty-block path"),
|job_id| {
assert_eq!(job_id, 4);
false
},
);
assert_eq!(failure.action, CREATE_JOB_POST);
assert_eq!(failure.nextJobID, 4);
assert_eq!(failure.jobReady, 1);
}
#[test]
fn flush_state_machine_preserves_offsets_and_completes_job() {
let job = [0x60u8, 0x61, 0x62, 0x63, 0x64, 0x65];
let mut output = [0xa5u8; 8];
let projection = ZSTDMT_flushJobProjection {
consumed: 4,
cSize: job.len(),
srcSize: 4,
dstStart: job.as_ptr().cast(),
dstCapacity: job.len(),
dstFlushed: 2,
frameChecksumNeeded: 0,
};
let context = ZSTDMT_flushContextProjection {
doneJobID: 0,
nextJobID: 1,
jobIDMask: 0,
..ZSTDMT_flushContextProjection::default()
};
let mut updated = None;
let mut completed = None;
let result = unsafe {
flush_produced_with(
context,
output.as_mut_ptr().cast(),
output.len(),
1,
0,
1,
|job_id, block_to_flush| {
assert_eq!(job_id, 0);
assert_eq!(block_to_flush, 0);
projection
},
|_job_id, _projection| panic!("unexpected checksum callback"),
|_job_id, dst_flushed| updated = Some(dst_flushed),
|_job_id, src_size, c_size| completed = Some((src_size, c_size)),
|| panic!("unexpected error callback"),
)
};
assert_eq!(result.result, 0);
assert_eq!(result.outputPos, 5);
assert_eq!(result.updateAllJobsCompleted, 1);
assert_eq!(result.allJobsCompleted, 0);
assert_eq!(updated, Some(6));
assert_eq!(completed, Some((4, 6)));
assert_eq!(&output[1..5], &job[2..]);
assert_eq!(&output[..1], &[0xa5]);
}
#[test]
fn flush_state_machine_keeps_pending_output_without_space() {
let job = [0x70u8, 0x71, 0x72, 0x73, 0x74, 0x75];
let mut output = [0xa5u8; 4];
let original = output;
let projection = ZSTDMT_flushJobProjection {
consumed: 2,
cSize: job.len(),
srcSize: 5,
dstStart: job.as_ptr().cast(),
dstCapacity: job.len(),
dstFlushed: 2,
frameChecksumNeeded: 0,
};
let context = ZSTDMT_flushContextProjection {
doneJobID: 3,
nextJobID: 4,
jobIDMask: 0,
..ZSTDMT_flushContextProjection::default()
};
let mut updated = None;
let mut completed = false;
let result = unsafe {
flush_produced_with(
context,
output.as_mut_ptr().cast(),
output.len(),
output.len(),
1,
1,
|_job_id, _block_to_flush| projection,
|_job_id, _projection| panic!("unexpected checksum callback"),
|_job_id, dst_flushed| updated = Some(dst_flushed),
|_job_id, _src_size, _c_size| completed = true,
|| panic!("unexpected error callback"),
)
};
assert_eq!(result.result, 4);
assert_eq!(result.outputPos, output.len());
assert_eq!(result.updateAllJobsCompleted, 0);
assert_eq!(updated, Some(2));
assert!(!completed);
assert_eq!(output, original);
}
#[test]
fn flush_state_machine_normalizes_worker_error_and_checksum_projection() {
let error_projection = ZSTDMT_flushJobProjection {
cSize: ERROR(ZstdErrorCode::Generic),
..ZSTDMT_flushJobProjection::default()
};
let context = ZSTDMT_flushContextProjection {
doneJobID: 0,
nextJobID: 1,
jobIDMask: 0,
..ZSTDMT_flushContextProjection::default()
};
let mut error_cleanup = false;
let result = unsafe {
flush_produced_with(
context,
ptr::null_mut(),
0,
0,
1,
1,
|_job_id, _block_to_flush| error_projection,
|_job_id, _projection| panic!("unexpected checksum callback"),
|_job_id, _dst_flushed| panic!("unexpected update callback"),
|_job_id, _src_size, _c_size| panic!("unexpected completion callback"),
|| error_cleanup = true,
)
};
assert_eq!(result.result, ERROR(ZstdErrorCode::Generic));
assert_eq!(result.outputPos, 0);
assert!(error_cleanup);
let job = [0x80u8, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86];
let mut output = [0xa5u8; 8];
let projection = ZSTDMT_flushJobProjection {
consumed: 4,
cSize: 3,
srcSize: 4,
dstStart: job.as_ptr().cast(),
dstCapacity: job.len(),
dstFlushed: 0,
frameChecksumNeeded: 1,
};
let mut checksum_calls = 0;
let mut completed_size = None;
let result = unsafe {
flush_produced_with(
ZSTDMT_flushContextProjection {
doneJobID: 0,
nextJobID: 1,
jobIDMask: 0,
..ZSTDMT_flushContextProjection::default()
},
output.as_mut_ptr().cast(),
output.len(),
0,
0,
1,
|_job_id, _block_to_flush| projection,
|_job_id, projected| {
checksum_calls += 1;
projected.cSize = 7;
projected.frameChecksumNeeded = 0;
},
|_job_id, _dst_flushed| {},
|_job_id, _src_size, c_size| completed_size = Some(c_size),
|| panic!("unexpected error callback"),
)
};
assert_eq!(checksum_calls, 1);
assert_eq!(completed_size, Some(7));
assert_eq!(result.result, 0);
assert_eq!(&output[..7], &job);
}
#[test]
fn flush_state_machine_propagates_checksum_projection_error() {
let job = [0x80u8, 0x81, 0x82, 0x83];
let mut output = [0xa5u8; 8];
let projection = ZSTDMT_flushJobProjection {
consumed: job.len(),
cSize: 3,
srcSize: job.len(),
dstStart: job.as_ptr().cast(),
dstCapacity: job.len(),
frameChecksumNeeded: 1,
..ZSTDMT_flushJobProjection::default()
};
let mut error_cleanup = false;
let mut completed = false;
let result = unsafe {
flush_produced_with(
ZSTDMT_flushContextProjection {
doneJobID: 0,
nextJobID: 1,
jobIDMask: 0,
..ZSTDMT_flushContextProjection::default()
},
output.as_mut_ptr().cast(),
output.len(),
0,
0,
1,
|_job_id, _block_to_flush| projection,
|_job_id, projected| {
projected.cSize = ERROR(ZstdErrorCode::DstSizeTooSmall);
},
|_job_id, _dst_flushed| panic!("unexpected update callback"),
|_job_id, _src_size, _c_size| completed = true,
|| error_cleanup = true,
)
};
assert_eq!(result.result, ERROR(ZstdErrorCode::DstSizeTooSmall));
assert_eq!(result.outputPos, 0);
assert!(error_cleanup);
assert!(!completed);
assert_eq!(output, [0xa5; 8]);
}
#[test]
fn frame_checksum_is_serialized_little_endian() {
let mut dst = [0xa5u8; 8];
let projection = ZSTDMT_frameChecksumProjection {
dst: dst.as_mut_ptr().cast(),
dstCapacity: dst.len(),
cSize: 4,
checksum: 0x7856_3412,
};
assert_eq!(write_frame_checksum(&projection), 8);
assert_eq!(&dst[..4], &[0xa5; 4]);
assert_eq!(&dst[4..], &[0x12, 0x34, 0x56, 0x78]);
}
#[test]
fn frame_checksum_accepts_exact_destination_capacity() {
let mut dst = [0xa5u8; 6];
let projection = ZSTDMT_frameChecksumProjection {
dst: dst.as_mut_ptr().cast(),
dstCapacity: dst.len(),
cSize: 2,
checksum: 0x1122_3344,
};
assert_eq!(write_frame_checksum(&projection), dst.len());
assert_eq!(&dst, &[0xa5, 0xa5, 0x44, 0x33, 0x22, 0x11]);
}
#[test]
fn frame_checksum_rejects_too_small_or_null_destination() {
let mut dst = [0xa5u8; 8];
let too_small = ZSTDMT_frameChecksumProjection {
dst: dst.as_mut_ptr().cast(),
dstCapacity: 7,
cSize: 4,
checksum: 0x0102_0304,
};
let null_destination = ZSTDMT_frameChecksumProjection {
dst: ptr::null_mut(),
dstCapacity: 4,
cSize: 0,
checksum: 0x0102_0304,
};
assert_eq!(
write_frame_checksum(&too_small),
ERROR(ZstdErrorCode::DstSizeTooSmall)
);
assert_eq!(
write_frame_checksum(&null_destination),
ERROR(ZstdErrorCode::DstSizeTooSmall)
);
assert_eq!(dst, [0xa5; 8]);
}
#[test]
fn flush_state_machine_reports_end_of_frame_state() {
let not_ended = unsafe {
flush_produced_with(
ZSTDMT_flushContextProjection {
frameEnded: 0,
..ZSTDMT_flushContextProjection::default()
},
ptr::null_mut(),
0,
0,
0,
ZSTD_E_END,
|_job_id, _block_to_flush| panic!("no job expected"),
|_job_id, _projection| panic!("no checksum expected"),
|_job_id, _dst_flushed| panic!("no update expected"),
|_job_id, _src_size, _c_size| panic!("no completion expected"),
|| panic!("no error expected"),
)
};
assert_eq!(not_ended.result, 1);
assert_eq!(not_ended.updateAllJobsCompleted, 1);
assert_eq!(not_ended.allJobsCompleted, 0);
let ended = unsafe {
flush_produced_with(
ZSTDMT_flushContextProjection {
frameEnded: 1,
..ZSTDMT_flushContextProjection::default()
},
ptr::null_mut(),
0,
0,
0,
ZSTD_E_END,
|_job_id, _block_to_flush| panic!("no job expected"),
|_job_id, _projection| panic!("no checksum expected"),
|_job_id, _dst_flushed| panic!("no update expected"),
|_job_id, _src_size, _c_size| panic!("no completion expected"),
|| panic!("no error expected"),
)
};
assert_eq!(ended.result, 0);
assert_eq!(ended.updateAllJobsCompleted, 1);
assert_eq!(ended.allJobsCompleted, 1);
}
#[test]
fn outer_scheduler_copies_input_and_reports_filled_after_progress() {
let input_bytes = [1u8, 2, 3, 4, 5];
let mut round_buffer = [0xa5u8; 4];
let mut output_buffer = [0xb6u8; 8];
let context = ZSTDMT_compressStreamContextProjection {
frameEnded: 0,
jobReady: 0,
inBuffStart: ptr::null_mut(),
inBuffCapacity: 0,
inBuffFilled: 0,
targetSectionSize: round_buffer.len(),
rsyncable: 0,
..ZSTDMT_compressStreamContextProjection::default()
};
let input = ZSTDMT_streamInputProjection {
src: input_bytes.as_ptr().cast(),
size: input_bytes.len(),
pos: 0,
};
let output = ZSTDMT_streamOutputProjection {
dst: output_buffer.as_mut_ptr().cast(),
size: output_buffer.len(),
pos: 1,
};
let mut range_calls = 0;
let mut create_calls = Vec::new();
let mut flush_calls = Vec::new();
let round_buffer_ptr = round_buffer.as_mut_ptr();
let round_capacity = round_buffer.len();
let result = unsafe {
compress_stream_generic_with(
context,
input,
output,
ZSTD_E_CONTINUE,
|projection| {
range_calls += 1;
projection.bufferStart = round_buffer_ptr.cast();
projection.bufferCapacity = round_capacity;
projection.bufferFilled = 0;
1
},
|src_size, end| {
create_calls.push((src_size, end));
0
},
|_dst, _size, output_pos, block_to_flush, end| {
flush_calls.push((block_to_flush, end));
ZSTDMT_streamFlushResult {
result: 0,
outputPos: output_pos + 2,
}
},
|_context, input, projection| ZSTDMT_syncPointProjection {
toLoad: (input.size - input.pos).min(round_capacity - projection.bufferFilled),
flush: 0,
},
)
};
assert_eq!(range_calls, 1);
assert_eq!(round_buffer, input_bytes[..round_buffer.len()]);
assert_eq!(create_calls, vec![(round_buffer.len(), ZSTD_E_CONTINUE)]);
assert_eq!(flush_calls, vec![(0, ZSTD_E_CONTINUE)]);
assert_eq!(result.result, 1);
assert_eq!(result.inputPos, round_buffer.len());
assert_eq!(result.outputPos, 3);
assert_eq!(result.inBuffFilled, round_buffer.len());
assert_eq!(output_buffer[0], 0xb6);
}
#[test]
fn outer_scheduler_forces_flush_before_end_when_input_remains() {
let input_bytes = [7u8, 8, 9, 10, 11];
let mut round_buffer = [0xa5u8; 3];
let context = ZSTDMT_compressStreamContextProjection {
inBuffStart: round_buffer.as_mut_ptr().cast(),
inBuffCapacity: round_buffer.len(),
targetSectionSize: round_buffer.len(),
rsyncable: 0,
..ZSTDMT_compressStreamContextProjection::default()
};
let input = ZSTDMT_streamInputProjection {
src: input_bytes.as_ptr().cast(),
size: input_bytes.len(),
pos: 0,
};
let output = ZSTDMT_streamOutputProjection::default();
let mut create_end = None;
let mut flush_end = None;
let mut flush_block = None;
let result = unsafe {
compress_stream_generic_with(
context,
input,
output,
ZSTD_E_END,
|_projection| panic!("input range should already be available"),
|src_size, end| {
create_end = Some((src_size, end));
0
},
|_dst, _size, output_pos, block_to_flush, end| {
flush_block = Some(block_to_flush);
flush_end = Some(end);
ZSTDMT_streamFlushResult {
result: 0,
outputPos: output_pos,
}
},
|_context, input, projection| ZSTDMT_syncPointProjection {
toLoad: (input.size - input.pos)
.min(round_buffer.len() - projection.bufferFilled),
flush: 0,
},
)
};
assert_eq!(create_end, Some((round_buffer.len(), ZSTD_E_FLUSH)));
assert_eq!(flush_end, Some(ZSTD_E_FLUSH));
assert_eq!(flush_block, Some(0));
assert_eq!(result.result, 1);
assert_eq!(result.inputPos, round_buffer.len());
}
#[test]
fn outer_scheduler_resubmits_ready_job_and_blocks_for_flush_progress() {
let context = ZSTDMT_compressStreamContextProjection {
jobReady: 1,
targetSectionSize: 4,
..ZSTDMT_compressStreamContextProjection::default()
};
let input_bytes = [0x31u8, 0x32];
let input = ZSTDMT_streamInputProjection {
src: input_bytes.as_ptr().cast(),
size: input_bytes.len(),
pos: 0,
};
let output = ZSTDMT_streamOutputProjection::default();
let mut create_call = None;
let mut flush_call = None;
let result = unsafe {
compress_stream_generic_with(
context,
input,
output,
ZSTD_E_CONTINUE,
|_projection| panic!("a ready job must skip input acquisition"),
|src_size, end| {
create_call = Some((src_size, end));
0
},
|_dst, _size, output_pos, block_to_flush, end| {
flush_call = Some((block_to_flush, end));
ZSTDMT_streamFlushResult {
result: 1,
outputPos: output_pos,
}
},
|_context, _input, _projection| panic!("ready job must skip sync scan"),
)
};
assert_eq!(create_call, Some((0, ZSTD_E_CONTINUE)));
assert_eq!(flush_call, Some((1, ZSTD_E_CONTINUE)));
assert_eq!(result.result, 1);
assert_eq!(result.inputPos, 0);
}
#[test]
fn outer_scheduler_stops_before_flush_on_job_error() {
let mut round_buffer = [0xa5u8; 2];
let input_bytes = [0x41u8, 0x42];
let context = ZSTDMT_compressStreamContextProjection {
inBuffStart: round_buffer.as_mut_ptr().cast(),
inBuffCapacity: round_buffer.len(),
targetSectionSize: round_buffer.len(),
..ZSTDMT_compressStreamContextProjection::default()
};
let input = ZSTDMT_streamInputProjection {
src: input_bytes.as_ptr().cast(),
size: input_bytes.len(),
pos: 0,
};
let output = ZSTDMT_streamOutputProjection::default();
let mut flush_called = false;
let result = unsafe {
compress_stream_generic_with(
context,
input,
output,
ZSTD_E_CONTINUE,
|_projection| panic!("input range should already be available"),
|_src_size, _end| ERROR(ZstdErrorCode::DstSizeTooSmall),
|_dst, _size, output_pos, _block_to_flush, _end| {
flush_called = true;
ZSTDMT_streamFlushResult {
result: 0,
outputPos: output_pos,
}
},
|_context, input, projection| ZSTDMT_syncPointProjection {
toLoad: (input.size - input.pos)
.min(round_buffer.len() - projection.bufferFilled),
flush: 0,
},
)
};
assert_eq!(result.result, ERROR(ZstdErrorCode::DstSizeTooSmall));
assert_eq!(result.inputPos, round_buffer.len());
assert_eq!(result.outputPos, 0);
assert!(!flush_called);
}
#[test]
fn empty_last_block_serializes_after_buffer_callback() {
let mut storage = vec![0xa5u8; 3];
let expected_start = storage.as_mut_ptr().cast::<c_void>();
let mut callback_calls = 0;
let projection = ZSTDMT_emptyBlockJobProjection {
lastJob: 1,
srcSize: 0,
firstJob: 0,
dstStart: ptr::null_mut(),
dstCapacity: 0,
consumed: 0,
};
let result = unsafe {
write_last_empty_block_with(projection, || {
callback_calls += 1;
ZstdMtBuffer {
start: expected_start,
capacity: storage.len(),
}
})
};
assert_eq!(callback_calls, 1);
assert_eq!(result.buffer.start, expected_start);
assert_eq!(result.buffer.capacity, storage.len());
assert_eq!(result.cSize, 3);
assert_eq!(result.clearSource, 1);
assert_eq!(storage, [1, 0, 0]);
}
#[test]
fn empty_last_block_keeps_source_on_buffer_allocation_failure() {
let projection = ZSTDMT_emptyBlockJobProjection {
lastJob: 1,
srcSize: 0,
firstJob: 0,
dstStart: ptr::null_mut(),
dstCapacity: 0,
consumed: 0,
};
let result = unsafe { write_last_empty_block_with(projection, ZstdMtBuffer::default) };
assert!(result.buffer.start.is_null());
assert_eq!(result.buffer.capacity, 0);
assert_eq!(result.cSize, ERROR(ZstdErrorCode::MemoryAllocation));
assert_eq!(result.clearSource, 0);
}
#[test]
fn chunk_loop_handles_empty_jobs_without_compression() {
let mut compressor = MockChunkCompressor {
continue_inputs: Vec::new(),
end_inputs: Vec::new(),
continue_results: Vec::new(),
end_results: vec![9],
};
let mut progress = MockChunkProgress::default();
let result = run_mock_chunk_loop(&mut compressor, &mut progress, 0, 8, 0);
assert_eq!(result, ZSTDMT_chunkProcessResult::default());
assert!(compressor.continue_inputs.is_empty());
assert!(compressor.end_inputs.is_empty());
assert!(progress.calls.is_empty());
let result = run_mock_chunk_loop(&mut compressor, &mut progress, 0, 8, 1);
assert_eq!(result.error, 0);
assert_eq!(result.lastBlockSize, 9);
assert!(compressor.continue_inputs.is_empty());
assert_eq!(compressor.end_inputs, [0]);
assert!(progress.calls.is_empty());
}
#[test]
fn chunk_loop_handles_one_exact_chunk_and_last_job_dispatch() {
let mut compressor = MockChunkCompressor {
continue_inputs: Vec::new(),
end_inputs: Vec::new(),
continue_results: vec![4],
end_results: vec![6],
};
let mut progress = MockChunkProgress::default();
let result = run_mock_chunk_loop(&mut compressor, &mut progress, 8, 8, 0);
assert_eq!(result.lastBlockSize, 4);
assert_eq!(compressor.continue_inputs, [8]);
assert!(compressor.end_inputs.is_empty());
assert!(progress.calls.is_empty());
let result = run_mock_chunk_loop(&mut compressor, &mut progress, 8, 8, 1);
assert_eq!(result.lastBlockSize, 6);
assert_eq!(compressor.end_inputs, [8]);
assert!(progress.calls.is_empty());
}
#[test]
fn chunk_loop_handles_multiple_exact_chunks_and_orders_progress() {
let mut compressor = MockChunkCompressor {
continue_inputs: Vec::new(),
end_inputs: Vec::new(),
continue_results: vec![2, 3],
end_results: Vec::new(),
};
let mut progress = MockChunkProgress::default();
let result = run_mock_chunk_loop(&mut compressor, &mut progress, 16, 8, 0);
assert_eq!(result.lastBlockSize, 3);
assert_eq!(compressor.continue_inputs, [8, 8]);
assert!(compressor.end_inputs.is_empty());
assert_eq!(progress.calls, [(2, 8)]);
}
#[test]
fn chunk_loop_handles_partial_tail_and_final_dispatch() {
let mut compressor = MockChunkCompressor {
continue_inputs: Vec::new(),
end_inputs: Vec::new(),
continue_results: vec![2],
end_results: vec![5],
};
let mut progress = MockChunkProgress::default();
let result = run_mock_chunk_loop(&mut compressor, &mut progress, 11, 8, 1);
assert_eq!(result.lastBlockSize, 5);
assert_eq!(compressor.continue_inputs, [8]);
assert_eq!(compressor.end_inputs, [3]);
assert_eq!(progress.calls, [(2, 8)]);
}
#[test]
fn chunk_loop_propagates_intermediate_errors_after_prior_progress() {
let error = ERROR(ZstdErrorCode::DstSizeTooSmall);
let mut compressor = MockChunkCompressor {
continue_inputs: Vec::new(),
end_inputs: Vec::new(),
continue_results: vec![2, error],
end_results: vec![7],
};
let mut progress = MockChunkProgress::default();
let result = run_mock_chunk_loop(&mut compressor, &mut progress, 24, 8, 1);
assert_eq!(result.error, error);
assert_eq!(result.lastBlockSize, 0);
assert_eq!(compressor.continue_inputs, [8, 8]);
assert!(compressor.end_inputs.is_empty());
assert_eq!(progress.calls, [(2, 8)]);
}
#[test]
fn chunk_loop_propagates_final_errors_without_progress_callback() {
let error = ERROR(ZstdErrorCode::Generic);
let mut compressor = MockChunkCompressor {
continue_inputs: Vec::new(),
end_inputs: Vec::new(),
continue_results: Vec::new(),
end_results: vec![error],
};
let mut progress = MockChunkProgress::default();
let result = run_mock_chunk_loop(&mut compressor, &mut progress, 3, 8, 1);
assert_eq!(result.error, error);
assert_eq!(result.lastBlockSize, 0);
assert!(compressor.continue_inputs.is_empty());
assert_eq!(compressor.end_inputs, [3]);
assert!(progress.calls.is_empty());
}
unsafe extern "C" fn probe_job_table_init(
_job_table: *mut c_void,
_nb_jobs: c_uint,
_job_size: usize,
) -> c_int {
JOB_TABLE_INIT_CALLS.fetch_add(1, Ordering::Relaxed);
if JOB_TABLE_FAIL_INIT.load(Ordering::Relaxed) {
-1
} else {
0
}
}
unsafe extern "C" fn probe_job_table_destroy(
_job_table: *mut c_void,
_nb_jobs: c_uint,
_job_size: usize,
) {
JOB_TABLE_DESTROY_CALLS.fetch_add(1, Ordering::Relaxed);
}
fn rsync_prime_power() -> u64 {
(0..RSYNC_LENGTH - 1).fold(1, |power, _| power.wrapping_mul(PRIME8_BYTES))
}
fn patterned_bytes(length: usize, seed: u8) -> Vec<u8> {
let mut value = seed;
let mut bytes = vec![0; length];
for byte in &mut bytes {
value = value.wrapping_mul(17).wrapping_add(29);
*byte = value;
}
bytes
}
fn rolling_hash_bytes(bytes: &[u8]) -> u64 {
bytes.iter().fold(0, |hash, &byte| {
hash.wrapping_mul(PRIME8_BYTES)
.wrapping_add(u64::from(byte).wrapping_add(ROLL_HASH_CHAR_OFFSET))
})
}
#[test]
fn rolling_hash_prime_power_matches_c_ipow_for_common_lengths() {
assert_eq!(ZSTDMT_rust_rollingHashPrimePower(1), 0x0000_0000_0000_0001);
assert_eq!(ZSTDMT_rust_rollingHashPrimePower(2), 0xCF1B_BCDC_B7A5_6463);
assert_eq!(
ZSTDMT_rust_rollingHashPrimePower(RSYNC_LENGTH as c_uint),
0xF550_7FE3_5F91_F8CB
);
}
#[test]
fn rolling_hash_prime_power_preserves_zero_length_underflow() {
assert_eq!(0u32.wrapping_sub(1), c_uint::MAX);
assert_eq!(ZSTDMT_rust_rollingHashPrimePower(0), 0x12A9_3A33_31E0_3D4B);
}
#[test]
fn frame_progression_zero_jobs_keeps_mt_base_values() {
assert_eq!(
ZSTDMT_rust_frameProgression(11, 7, 13, 4),
ZSTD_frameProgression {
ingested: 18,
consumed: 11,
produced: 13,
flushed: 13,
currentJobID: 4,
nbActiveWorkers: 0,
}
);
}
#[test]
fn frame_progression_one_ready_job_counts_as_active() {
let base = ZSTDMT_rust_frameProgression(100, 20, 300, 9);
assert_eq!(
ZSTDMT_rust_frameProgressionAddJob(base, 80, 17, 42, 10),
ZSTD_frameProgression {
ingested: 200,
consumed: 117,
produced: 342,
flushed: 310,
currentJobID: 9,
nbActiveWorkers: 1,
}
);
}
#[test]
fn frame_progression_completed_job_is_not_active() {
let base = ZSTDMT_rust_frameProgression(100, 20, 300, 9);
let progression = ZSTDMT_rust_frameProgressionAddJob(base, 80, 80, 42, 42);
assert_eq!(progression.nbActiveWorkers, 0);
assert_eq!(progression.ingested, 200);
assert_eq!(progression.consumed, 180);
}
#[test]
fn frame_progression_error_job_uses_normalized_zero_output() {
let base = ZSTDMT_rust_frameProgression(100, 20, 300, 9);
let progression = ZSTDMT_rust_frameProgressionAddJob(base, 80, 0, 0, 0);
assert_eq!(progression.produced, 300);
assert_eq!(progression.flushed, 300);
assert_eq!(progression.nbActiveWorkers, 1);
}
#[test]
fn frame_progression_aggregates_active_and_completed_jobs() {
let base = ZSTDMT_rust_frameProgression(100, 20, 300, 9);
let progression = ZSTDMT_rust_frameProgressionAddJob(base, 80, 17, 42, 10);
let progression = ZSTDMT_rust_frameProgressionAddJob(progression, 30, 30, 8, 8);
let progression = ZSTDMT_rust_frameProgressionAddJob(progression, 40, 5, 7, 6);
assert_eq!(
progression,
ZSTD_frameProgression {
ingested: 270,
consumed: 152,
produced: 357,
flushed: 324,
currentJobID: 9,
nbActiveWorkers: 2,
}
);
}
#[test]
fn frame_progression_with_jobs_masks_slots_and_normalizes_errors() {
let projections = [
ZSTDMT_jobProjection {
consumed: 50,
cSize: 8,
srcSize: 50,
dstFlushed: 8,
..ZSTDMT_jobProjection::default()
},
ZSTDMT_jobProjection {
consumed: 17,
cSize: 42,
srcSize: 80,
dstFlushed: 10,
..ZSTDMT_jobProjection::default()
},
ZSTDMT_jobProjection {
consumed: 30,
cSize: ERROR(ZstdErrorCode::SequenceProducerFailed),
srcSize: 30,
dstFlushed: 29,
..ZSTDMT_jobProjection::default()
},
ZSTDMT_jobProjection {
consumed: 5,
cSize: 7,
srcSize: 40,
dstFlushed: 6,
..ZSTDMT_jobProjection::default()
},
];
let mut calls = Vec::new();
let progression = frame_progression_with_jobs(100, 20, 300, 9, 5, 8, 1, 3, |job_id| {
calls.push(job_id);
projections[job_id as usize]
});
assert_eq!(calls, [1, 2, 3, 0]);
assert_eq!(
progression,
ZSTD_frameProgression {
ingested: 320,
consumed: 202,
produced: 357,
flushed: 324,
currentJobID: 9,
nbActiveWorkers: 2,
}
);
}
#[test]
fn input_data_in_use_skips_first_round_and_scans_oldest_ring_slot() {
let source_a = [0u8; 8];
let backing = [1u8; 16];
let source_b = &backing[8..];
let prefix_b = &backing[..3];
let projections = [
ZSTDMT_jobProjection {
consumed: source_a.len(),
srcStart: source_a.as_ptr().cast(),
srcSize: source_a.len(),
..ZSTDMT_jobProjection::default()
},
ZSTDMT_jobProjection {
consumed: 2,
srcStart: source_b.as_ptr().cast(),
srcSize: source_b.len(),
prefixStart: prefix_b.as_ptr().cast(),
prefixSize: prefix_b.len(),
..ZSTDMT_jobProjection::default()
},
];
let mut calls = 0;
let result = get_input_data_in_use_with(4, 6, 1, 16, 8, |slot| {
calls += 1;
projections[slot as usize]
});
assert_eq!(calls, 2);
assert_eq!(result.start, prefix_b.as_ptr().cast());
assert_eq!(result.size, prefix_b.len());
let mut first_round_calls = 0;
let result = get_input_data_in_use_with(0, 3, 1, 32, 8, |slot| {
first_round_calls += 1;
projections[slot as usize]
});
assert_eq!(first_round_calls, 0);
assert_eq!(result, ZSTDMT_inputRange::default());
}
#[test]
fn input_data_in_use_falls_back_to_source_without_prefix() {
let source = [3u8; 11];
let projection = ZSTDMT_jobProjection {
consumed: 4,
srcStart: source.as_ptr().cast(),
srcSize: source.len(),
..ZSTDMT_jobProjection::default()
};
let result = get_input_data_in_use_with(7, 8, 7, 8, 8, |_| projection);
assert_eq!(result.start, source.as_ptr().cast());
assert_eq!(result.size, source.len());
}
#[test]
fn try_get_input_range_selects_next_section_without_wrapping() {
let round_buffer = [0u8; 32];
let prefix = [1u8; 4];
let projection = ZSTDMT_tryGetInputRangeProjection {
roundBufferStart: round_buffer.as_ptr().cast(),
roundBufferCapacity: round_buffer.len(),
roundBufferPos: 8,
prefixStart: prefix.as_ptr().cast(),
prefixSize: prefix.len(),
targetSectionSize: 8,
..ZSTDMT_tryGetInputRangeProjection::default()
};
let result = try_get_input_range_with(projection);
assert_eq!(result.ready, 1);
assert_eq!(result.movePrefix, 0);
assert_eq!(result.roundBufferPos, 8);
assert_eq!(
result.bufferStart,
round_buffer.as_ptr().wrapping_add(8).cast_mut().cast()
);
assert_eq!(result.bufferCapacity, 8);
}
#[test]
fn try_get_input_range_wraps_after_prefix_and_rejects_live_ranges() {
let round_buffer = [0u8; 32];
let prefix = [1u8; 4];
let wrapped = ZSTDMT_tryGetInputRangeProjection {
roundBufferStart: round_buffer.as_ptr().cast(),
roundBufferCapacity: round_buffer.len(),
roundBufferPos: 28,
prefixStart: prefix.as_ptr().cast(),
prefixSize: prefix.len(),
targetSectionSize: 8,
..ZSTDMT_tryGetInputRangeProjection::default()
};
let result = try_get_input_range_with(wrapped);
assert_eq!(result.ready, 1);
assert_eq!(result.movePrefix, 1);
assert_eq!(result.roundBufferPos, prefix.len());
assert_eq!(
result.bufferStart,
round_buffer.as_ptr().wrapping_add(4).cast_mut().cast()
);
let blocked_prefix = ZSTDMT_tryGetInputRangeProjection {
inUseStart: round_buffer.as_ptr().cast(),
inUseSize: prefix.len(),
..wrapped
};
assert_eq!(try_get_input_range_with(blocked_prefix).ready, 0);
let blocked_source = ZSTDMT_tryGetInputRangeProjection {
roundBufferPos: 8,
inUseStart: round_buffer.as_ptr().wrapping_add(8).cast(),
inUseSize: 8,
..wrapped
};
assert_eq!(try_get_input_range_with(blocked_source).ready, 0);
}
#[test]
fn to_flush_now_projects_oldest_slot_and_normalizes_errors() {
let projection = ZSTDMT_jobProjection {
consumed: 5,
cSize: 23,
srcSize: 17,
dstFlushed: 9,
..ZSTDMT_jobProjection::default()
};
let mut seen_slot = None;
let to_flush = to_flush_now_with(5, 9, 3, |slot| {
seen_slot = Some(slot);
projection
});
assert_eq!(seen_slot, Some(1));
assert_eq!(to_flush, 14);
let error_projection = ZSTDMT_jobProjection {
consumed: 0,
cSize: ERROR(ZstdErrorCode::Generic),
srcSize: 17,
dstFlushed: 23,
..ZSTDMT_jobProjection::default()
};
assert_eq!(to_flush_now_with(1, 2, 1, |_| error_projection), 0);
let mut called = false;
assert_eq!(
to_flush_now_with(2, 2, 1, |_| {
called = true;
projection
}),
0
);
assert!(!called);
}
fn call_synchronization_point(
input: &[u8],
input_pos: usize,
in_buff: &[u8],
in_buff_filled: usize,
target_section_size: usize,
rsyncable: c_int,
hit_mask: u64,
) -> (usize, c_int) {
let mut to_load = usize::MAX;
let mut flush = -1;
unsafe {
ZSTDMT_rust_findSynchronizationPoint(
input.as_ptr().cast(),
input.len(),
input_pos,
target_section_size,
in_buff.as_ptr().cast(),
in_buff_filled,
rsyncable,
rsync_prime_power(),
hit_mask,
&mut to_load,
&mut flush,
);
}
(to_load, flush)
}
fn input_hashes_after_each_byte(
in_buff: &[u8],
in_buff_filled: usize,
input: &[u8],
input_pos: usize,
) -> (u64, Vec<u64>) {
let mut history = in_buff[in_buff_filled - RSYNC_LENGTH..in_buff_filled].to_vec();
history.extend_from_slice(&input[input_pos..]);
let initial = rolling_hash_bytes(&history[..RSYNC_LENGTH]);
let hashes = (0..input.len() - input_pos)
.map(|pos| rolling_hash_bytes(&history[pos + 1..pos + 1 + RSYNC_LENGTH]))
.collect();
(initial, hashes)
}
fn hit_mask_for_index(initial: u64, hashes: &[u64], target: usize) -> u64 {
let target_hash = hashes[target];
let prior_matches = |mask: u64| {
(initial & mask) == mask || hashes[..target].iter().any(|hash| (hash & mask) == mask)
};
if target_hash != 0 && !prior_matches(target_hash) {
return target_hash;
}
for mask in 1..=u64::from(u16::MAX) {
if target_hash & mask == mask && !prior_matches(mask) {
return mask;
}
}
panic!("could not isolate synchronization hash at position {target}");
}
#[test]
fn synchronization_point_keeps_rsync_disabled() {
let input = patterned_bytes(20, 3);
let in_buff = patterned_bytes(64, 5);
assert_eq!(
call_synchronization_point(&input, 3, &in_buff, 40, 100, 0, 0),
(17, 0)
);
}
#[test]
fn synchronization_point_does_not_scan_a_short_hash_window() {
let input = patterned_bytes(RSYNC_LENGTH - 1, 7);
let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 11);
assert_eq!(
call_synchronization_point(
&input,
0,
&in_buff,
0,
RSYNC_MIN_BLOCK_SIZE + input.len(),
1,
0,
),
(input.len(), 0)
);
}
#[test]
fn synchronization_point_respects_min_block_boundaries() {
let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 13);
for &in_buff_filled in &[
0,
RSYNC_LENGTH - 1,
RSYNC_LENGTH,
RSYNC_MIN_BLOCK_SIZE - 1,
RSYNC_MIN_BLOCK_SIZE,
] {
let input_len = if in_buff_filled < RSYNC_MIN_BLOCK_SIZE {
RSYNC_MIN_BLOCK_SIZE - in_buff_filled + 1
} else {
1
};
let input = patterned_bytes(input_len, in_buff_filled as u8);
let expected = if in_buff_filled == RSYNC_MIN_BLOCK_SIZE {
(0, 1)
} else {
(input_len, 1)
};
assert_eq!(
call_synchronization_point(
&input,
0,
&in_buff,
in_buff_filled,
in_buff_filled + input_len,
1,
0,
),
expected,
"inBuff.filled={in_buff_filled}"
);
}
}
#[test]
fn synchronization_point_reports_initial_first_middle_and_final_hits() {
let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 17);
let input = patterned_bytes(96, 19);
let (initial, hashes) =
input_hashes_after_each_byte(&in_buff, RSYNC_MIN_BLOCK_SIZE, &input, 0);
assert_eq!(
call_synchronization_point(
&input,
0,
&in_buff,
RSYNC_MIN_BLOCK_SIZE,
RSYNC_MIN_BLOCK_SIZE + input.len(),
1,
0,
),
(0, 1)
);
for &(name, index) in &[("first", 0), ("middle", 37), ("final", 95)] {
let hit_mask = hit_mask_for_index(initial, &hashes, index);
assert_eq!(
call_synchronization_point(
&input,
0,
&in_buff,
RSYNC_MIN_BLOCK_SIZE,
RSYNC_MIN_BLOCK_SIZE + input.len(),
1,
hit_mask,
),
(index + 1, 1),
"{name} hit"
);
}
}
#[test]
fn synchronization_point_loads_all_input_when_no_hash_hits() {
let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 23);
let input = patterned_bytes(96, 29);
let (initial, hashes) =
input_hashes_after_each_byte(&in_buff, RSYNC_MIN_BLOCK_SIZE, &input, 0);
assert_ne!(initial, u64::MAX);
assert!(hashes.iter().all(|&hash| hash != u64::MAX));
assert_eq!(
call_synchronization_point(
&input,
0,
&in_buff,
RSYNC_MIN_BLOCK_SIZE,
RSYNC_MIN_BLOCK_SIZE + input.len(),
1,
u64::MAX,
),
(input.len(), 0)
);
}
#[test]
fn synchronization_point_preserves_nonzero_input_position() {
let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 31);
let prefix = patterned_bytes(7, 37);
let payload = patterned_bytes(64, 41);
let mut input = prefix;
input.extend_from_slice(&payload);
let (initial, hashes) =
input_hashes_after_each_byte(&in_buff, RSYNC_MIN_BLOCK_SIZE, &input, 7);
let hit_mask = hit_mask_for_index(initial, &hashes, 23);
assert_eq!(
call_synchronization_point(
&input,
7,
&in_buff,
RSYNC_MIN_BLOCK_SIZE,
RSYNC_MIN_BLOCK_SIZE + payload.len(),
1,
hit_mask,
),
(24, 1)
);
}
#[test]
fn synchronization_point_handles_a_window_spanning_the_buffer_boundary() {
let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 43);
let input = [0xe1, 0x7a];
let in_buff_filled = RSYNC_MIN_BLOCK_SIZE - 1;
let mut history = in_buff[in_buff_filled - RSYNC_LENGTH..in_buff_filled].to_vec();
history.extend_from_slice(&input);
let hit_mask = rolling_hash_bytes(&history[2..2 + RSYNC_LENGTH]);
assert_eq!(
call_synchronization_point(
&input,
0,
&in_buff,
in_buff_filled,
in_buff_filled + input.len(),
1,
hit_mask,
),
(input.len(), 1)
);
}
#[test]
fn raw_seq_buffer_conversion_uses_whole_element_capacity() {
let mut sequences = [ZstdMtRawSeq::default(); 3];
let element_size = mem::size_of::<ZstdMtRawSeq>();
let buffer = ZstdMtBuffer {
start: sequences.as_mut_ptr().cast(),
capacity: element_size * sequences.len() + element_size - 1,
};
let seq = ZSTDMT_rust_bufferToSeq(buffer);
assert_eq!(seq.seq, sequences.as_mut_ptr());
assert_eq!(seq.pos, 0);
assert_eq!(seq.posInSequence, 0);
assert_eq!(seq.size, 0);
assert_eq!(seq.capacity, sequences.len());
let roundtrip = ZSTDMT_rust_seqToBuffer(seq);
assert_eq!(roundtrip.start, buffer.start);
assert_eq!(roundtrip.capacity, element_size * sequences.len());
}
#[test]
fn raw_seq_to_buffer_preserves_pointer_and_size_t_multiplication() {
let seq = ZstdMtRawSeqStore {
seq: ptr::null_mut(),
pos: 4,
posInSequence: 5,
size: 6,
capacity: usize::MAX,
};
let buffer = ZSTDMT_rust_seqToBuffer(seq);
assert!(buffer.start.is_null());
assert_eq!(
buffer.capacity,
usize::MAX.wrapping_mul(mem::size_of::<ZstdMtRawSeq>())
);
let empty = ZSTDMT_rust_bufferToSeq(ZstdMtBuffer {
start: ptr::null_mut(),
capacity: mem::size_of::<ZstdMtRawSeq>() - 1,
});
assert!(empty.seq.is_null());
assert_eq!(empty.capacity, 0);
assert_eq!(empty.pos, 0);
assert_eq!(empty.posInSequence, 0);
assert_eq!(empty.size, 0);
}
#[test]
fn overlapped_rejects_null_ranges() {
let bytes = [0u8; 8];
let start = bytes.as_ptr().cast::<c_void>();
assert_eq!(ZSTDMT_rust_isOverlapped(ptr::null(), 4, start, 4), 0);
assert_eq!(ZSTDMT_rust_isOverlapped(start, 4, ptr::null(), 4), 0);
}
#[test]
fn overlapped_rejects_empty_ranges() {
let bytes = [0u8; 8];
let start = bytes.as_ptr().cast::<c_void>();
assert_eq!(ZSTDMT_rust_isOverlapped(start, 0, start, 4), 0);
assert_eq!(ZSTDMT_rust_isOverlapped(start, 4, start, 0), 0);
}
#[test]
fn overlapped_uses_half_open_range_boundaries() {
let bytes = [0u8; 16];
let start = bytes.as_ptr();
let middle = start.wrapping_add(4);
assert_eq!(
ZSTDMT_rust_isOverlapped(start.cast(), 4, middle.cast(), 4),
0
);
assert_eq!(
ZSTDMT_rust_isOverlapped(middle.cast(), 4, start.cast(), 4),
0
);
}
#[test]
fn overlapped_detects_contained_ranges() {
let bytes = [0u8; 16];
let start = bytes.as_ptr();
let inner = start.wrapping_add(4);
assert_eq!(
ZSTDMT_rust_isOverlapped(start.cast(), 12, inner.cast(), 4),
1
);
assert_eq!(
ZSTDMT_rust_isOverlapped(inner.cast(), 4, start.cast(), 12),
1
);
}
#[test]
fn overlapped_rejects_disjoint_ranges() {
let bytes = [0u8; 16];
let start = bytes.as_ptr();
let after = start.wrapping_add(8);
assert_eq!(
ZSTDMT_rust_isOverlapped(start.cast(), 4, after.cast(), 4),
0
);
assert_eq!(
ZSTDMT_rust_isOverlapped(after.cast(), 4, start.cast(), 4),
0
);
}
#[test]
fn overlap_window_checks_external_dictionary_and_prefix() {
let bytes = [0u8; 32];
let base = bytes.as_ptr();
let next_src = base.wrapping_add(16);
let dict_base = base.wrapping_add(16);
assert_eq!(
ZSTDMT_rust_doesOverlapWindow(
base.wrapping_add(20).cast(),
4,
next_src.cast(),
base.cast(),
dict_base.cast(),
8,
4,
),
1
);
assert_eq!(
ZSTDMT_rust_doesOverlapWindow(
base.wrapping_add(12).cast(),
4,
next_src.cast(),
base.cast(),
dict_base.cast(),
8,
4,
),
1
);
assert_eq!(
ZSTDMT_rust_doesOverlapWindow(
base.cast(),
4,
next_src.cast(),
base.cast(),
dict_base.cast(),
8,
4,
),
0
);
}
#[test]
fn direct_overlap_window_abi_matches_scalar_policy() {
let bytes = [0u8; 32];
let base = bytes.as_ptr();
let window = ZstdMtWindow {
nextSrc: base.wrapping_add(16),
base,
dictBase: base.wrapping_add(16),
dictLimit: 8,
lowLimit: 4,
nbOverflowCorrections: 0,
};
let buffer = ZstdMtBuffer {
start: base.wrapping_add(20) as *mut c_void,
capacity: 4,
};
assert_eq!(ZSTDMT_doesOverlapWindow(buffer, window), 1);
assert_eq!(
ZSTDMT_doesOverlapWindow(
ZstdMtBuffer {
start: base as *mut u8 as *mut c_void,
capacity: 4,
},
window,
),
0
);
}
#[test]
fn buffer_pool_reuses_and_resizes_buffers() {
let pool = unsafe { ZSTDMT_rust_buffer_pool_create(2, DEFAULT_MEM) };
assert!(!pool.is_null());
let first = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
assert!(!first.start.is_null());
assert_eq!(first.capacity, 64 << 10);
unsafe { ZSTDMT_rust_buffer_pool_release(pool, first) };
let reused = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
assert_eq!(reused.start, first.start);
assert_eq!(reused.capacity, first.capacity);
unsafe { ZSTDMT_rust_buffer_pool_release(pool, reused) };
unsafe { ZSTDMT_rust_buffer_pool_set_size(pool, 128 << 10) };
let resized = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
assert_eq!(resized.capacity, 128 << 10);
unsafe {
ZSTDMT_rust_buffer_pool_release(pool, resized);
ZSTDMT_rust_buffer_pool_free(pool);
}
}
#[test]
fn zero_size_pool_does_not_allocate_sequence_storage() {
let pool = unsafe { ZSTDMT_rust_buffer_pool_create(1, DEFAULT_MEM) };
assert!(!pool.is_null());
unsafe { ZSTDMT_rust_buffer_pool_set_size(pool, 0) };
let buffer = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
assert!(buffer.start.is_null());
assert_eq!(buffer.capacity, 0);
unsafe { ZSTDMT_rust_buffer_pool_free(pool) };
}
#[test]
fn expansion_preserves_requested_buffer_size() {
let pool = unsafe { ZSTDMT_rust_buffer_pool_create(1, DEFAULT_MEM) };
assert!(!pool.is_null());
unsafe { ZSTDMT_rust_buffer_pool_set_size(pool, 96 << 10) };
let expanded = unsafe { ZSTDMT_rust_buffer_pool_expand(pool, 3) };
assert!(!expanded.is_null());
let buffer = unsafe { ZSTDMT_rust_buffer_pool_get(expanded) };
assert_eq!(buffer.capacity, 96 << 10);
unsafe {
ZSTDMT_rust_buffer_pool_release(expanded, buffer);
ZSTDMT_rust_buffer_pool_free(expanded);
}
}
#[test]
fn resize_preserves_existing_buffer_contents() {
let pool = unsafe { ZSTDMT_rust_buffer_pool_create(1, DEFAULT_MEM) };
assert!(!pool.is_null());
let buffer = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
assert!(!buffer.start.is_null());
let sample = b"multithreaded-buffer-pool";
unsafe {
ptr::copy_nonoverlapping(sample.as_ptr(), buffer.start.cast(), sample.len());
ZSTDMT_rust_buffer_pool_set_size(pool, buffer.capacity + 1);
}
let resized = unsafe { ZSTDMT_rust_buffer_pool_resize(pool, buffer) };
assert_eq!(resized.capacity, buffer.capacity + 1);
let contents =
unsafe { std::slice::from_raw_parts(resized.start.cast::<u8>(), sample.len()) };
assert_eq!(contents, sample);
unsafe {
ZSTDMT_rust_buffer_pool_release(pool, resized);
ZSTDMT_rust_buffer_pool_free(pool);
}
}
#[test]
fn job_table_count_matches_c_power_of_two_contract() {
assert_eq!(rounded_job_count(0), None);
assert_eq!(rounded_job_count(1), Some(2));
assert_eq!(rounded_job_count(3), Some(4));
assert_eq!(rounded_job_count(4), Some(8));
assert_eq!(rounded_job_count(255), Some(256));
assert_eq!(rounded_job_count(256), Some(512));
}
#[test]
fn job_table_creation_cleans_up_after_sync_init_failure() {
let _lock = JOB_TABLE_TEST_LOCK.lock().unwrap();
JOB_TABLE_INIT_CALLS.store(0, Ordering::Relaxed);
JOB_TABLE_DESTROY_CALLS.store(0, Ordering::Relaxed);
JOB_TABLE_FAIL_INIT.store(true, Ordering::Relaxed);
let mut requested_jobs = 1;
let job_table = unsafe {
ZSTDMT_rust_createJobsTable(
&mut requested_jobs,
1,
DEFAULT_MEM,
probe_job_table_init,
probe_job_table_destroy,
)
};
assert!(job_table.is_null());
assert_eq!(requested_jobs, 2);
assert_eq!(JOB_TABLE_INIT_CALLS.load(Ordering::Relaxed), 1);
assert_eq!(JOB_TABLE_DESTROY_CALLS.load(Ordering::Relaxed), 1);
JOB_TABLE_FAIL_INIT.store(false, Ordering::Relaxed);
}
#[test]
fn job_table_expansion_preserves_mask_and_sync_lifecycle() {
let _lock = JOB_TABLE_TEST_LOCK.lock().unwrap();
JOB_TABLE_INIT_CALLS.store(0, Ordering::Relaxed);
JOB_TABLE_DESTROY_CALLS.store(0, Ordering::Relaxed);
JOB_TABLE_FAIL_INIT.store(false, Ordering::Relaxed);
let mut requested_jobs = 1;
let initial_table =
unsafe { ZSTDMT_rust_job_table_create(&mut requested_jobs, 1, DEFAULT_MEM) };
assert!(!initial_table.is_null());
assert_eq!(requested_jobs, 2);
let mut job_table = initial_table;
let mut job_id_mask = 1;
assert_eq!(
unsafe {
ZSTDMT_rust_expandJobsTable(
&mut job_table,
&mut job_id_mask,
2,
1,
DEFAULT_MEM,
probe_job_table_init,
probe_job_table_destroy,
)
},
0
);
assert!(!job_table.is_null());
assert_eq!(job_id_mask, 7);
assert_eq!(JOB_TABLE_INIT_CALLS.load(Ordering::Relaxed), 1);
assert_eq!(JOB_TABLE_DESTROY_CALLS.load(Ordering::Relaxed), 1);
let expanded_table = job_table;
assert_eq!(
unsafe {
ZSTDMT_rust_expandJobsTable(
&mut job_table,
&mut job_id_mask,
2,
1,
DEFAULT_MEM,
probe_job_table_init,
probe_job_table_destroy,
)
},
0
);
assert_eq!(job_table, expanded_table);
assert_eq!(job_id_mask, 7);
assert_eq!(JOB_TABLE_INIT_CALLS.load(Ordering::Relaxed), 1);
assert_eq!(JOB_TABLE_DESTROY_CALLS.load(Ordering::Relaxed), 1);
unsafe { ZSTDMT_rust_job_table_free(job_table, 8, 1, DEFAULT_MEM) };
}
#[test]
fn job_table_expansion_cleans_up_after_sync_init_failure() {
let _lock = JOB_TABLE_TEST_LOCK.lock().unwrap();
JOB_TABLE_INIT_CALLS.store(0, Ordering::Relaxed);
JOB_TABLE_DESTROY_CALLS.store(0, Ordering::Relaxed);
JOB_TABLE_FAIL_INIT.store(true, Ordering::Relaxed);
let mut requested_jobs = 1;
let initial_table =
unsafe { ZSTDMT_rust_job_table_create(&mut requested_jobs, 1, DEFAULT_MEM) };
assert!(!initial_table.is_null());
let mut job_table = initial_table;
let mut job_id_mask = 1;
assert_eq!(
unsafe {
ZSTDMT_rust_expandJobsTable(
&mut job_table,
&mut job_id_mask,
2,
1,
DEFAULT_MEM,
probe_job_table_init,
probe_job_table_destroy,
)
},
ERROR(ZstdErrorCode::MemoryAllocation)
);
assert!(job_table.is_null());
assert_eq!(job_id_mask, 0);
assert_eq!(JOB_TABLE_INIT_CALLS.load(Ordering::Relaxed), 1);
assert_eq!(JOB_TABLE_DESTROY_CALLS.load(Ordering::Relaxed), 2);
JOB_TABLE_FAIL_INIT.store(false, Ordering::Relaxed);
}
#[test]
fn target_job_log_preserves_ldm_and_non_ldm_policy() {
assert_eq!(
compute_target_job_log(18, 25, ZSTD_FAST, ZSTD_PS_DISABLE),
20
);
assert_eq!(
compute_target_job_log(30, 25, ZSTD_FAST, ZSTD_PS_DISABLE),
ZSTDMT_JOBLOG_MAX
);
assert_eq!(
compute_target_job_log(10, 25, ZSTD_FAST, ZSTD_PS_ENABLE),
28
);
assert_eq!(
compute_target_job_log(30, 25, ZSTD_BTULTRA2, ZSTD_PS_ENABLE),
27
);
assert_eq!(
compute_target_job_log(10, 30, ZSTD_BTULTRA2, ZSTD_PS_ENABLE),
ZSTDMT_JOBLOG_MAX
);
}
#[test]
fn compression_job_parameters_disable_worker_owned_features() {
assert_eq!(
prepare_compression_job_parameters(0, 1, ZSTD_PS_ENABLE, 4),
ZSTDMT_compressionJobParameters {
checksumFlag: 1,
ldmEnable: ZSTD_PS_DISABLE,
nbWorkers: 0,
}
);
}
#[test]
fn compression_job_parameters_clear_checksum_after_first_job() {
assert_eq!(
prepare_compression_job_parameters(3, 1, ZSTD_PS_ENABLE, 4),
ZSTDMT_compressionJobParameters {
checksumFlag: 0,
ldmEnable: ZSTD_PS_DISABLE,
nbWorkers: 0,
}
);
assert_eq!(
prepare_compression_job_parameters(9, 0, ZSTD_PS_DISABLE, 0),
ZSTDMT_compressionJobParameters {
checksumFlag: 0,
ldmEnable: ZSTD_PS_DISABLE,
nbWorkers: 0,
}
);
}
#[test]
fn overlap_log_defaults_follow_strategy_groups() {
assert_eq!(overlap_log_default(ZSTD_FAST), 6);
assert_eq!(overlap_log_default(ZSTD_DFAST), 6);
assert_eq!(overlap_log_default(ZSTD_GREEDY), 6);
assert_eq!(overlap_log_default(ZSTD_LAZY), 6);
assert_eq!(overlap_log_default(ZSTD_LAZY2), 7);
assert_eq!(overlap_log_default(ZSTD_BTLAZY2), 7);
assert_eq!(overlap_log_default(ZSTD_BTOPT), 8);
assert_eq!(overlap_log_default(ZSTD_BTULTRA), 8);
assert_eq!(overlap_log_default(ZSTD_BTULTRA2), 9);
assert_eq!(overlap_log_default(0), 6);
assert_eq!(overlap_log(0, ZSTD_BTULTRA2), 9);
assert_eq!(overlap_log(0, ZSTD_FAST), 6);
assert_eq!(overlap_log(5, ZSTD_BTULTRA2), 5);
assert_eq!(overlap_log(9, ZSTD_FAST), 9);
}
#[test]
fn overlap_size_uses_window_or_target_job_log_as_expected() {
assert_eq!(
compute_overlap_size(20, 25, ZSTD_FAST, 0, ZSTD_PS_DISABLE),
1usize << 17
);
assert_eq!(
compute_overlap_size(20, 25, ZSTD_BTULTRA2, 0, ZSTD_PS_DISABLE),
1usize << 20
);
assert_eq!(
compute_overlap_size(20, 25, ZSTD_FAST, 1, ZSTD_PS_DISABLE),
0
);
assert_eq!(
compute_overlap_size(30, 25, ZSTD_FAST, 0, ZSTD_PS_ENABLE),
1usize << 23
);
assert_eq!(
compute_overlap_size(30, 25, ZSTD_BTULTRA2, 0, ZSTD_PS_ENABLE),
1usize << 25
);
}
#[test]
fn cctx_pool_size_addition_handles_zero_components() {
assert_eq!(sizeof_cctx_pool(0, 0), 0);
assert_eq!(sizeof_cctx_pool(37, 0), 37);
assert_eq!(sizeof_cctx_pool(0, 53), 53);
}
#[test]
fn cctx_pool_size_addition_preserves_ordinary_sum() {
assert_eq!(sizeof_cctx_pool(128, 4096), 4224);
}
#[test]
fn cctx_pool_size_addition_wraps_like_c_size_t() {
assert_eq!(sizeof_cctx_pool(usize::MAX, 1), 0);
assert_eq!(sizeof_cctx_pool(usize::MAX - 7, 11), 3);
}
#[test]
fn buffer_pool_size_addition_handles_zero_components() {
assert_eq!(sizeof_buffer_pool(0, 0), 0);
assert_eq!(sizeof_buffer_pool(37, 0), 37);
assert_eq!(sizeof_buffer_pool(0, 53), 53);
}
#[test]
fn buffer_pool_size_addition_preserves_ordinary_sum() {
assert_eq!(sizeof_buffer_pool(128, 4096), 4224);
}
#[test]
fn buffer_pool_size_addition_wraps_like_c_size_t() {
assert_eq!(sizeof_buffer_pool(usize::MAX, 1), 0);
assert_eq!(sizeof_buffer_pool(usize::MAX - 7, 11), 3);
}
}
Reference in New Issue View Git Blame Copy Permalink