zstd-rs/rust/src/zstdmt_compress.rs

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]

//! Rust-owned resource pools used by the multithreaded compressor.
//!
//! The serial LDM state, job descriptor fields, worker callback, and streaming
//! state still use private C layouts.  `zstdmt_compress.c` therefore keeps
//! those operations and projects allocation/lifecycle pieces, dictionary
//! transition policy, and pure sizing policy into this module.  The entry
//! points below are narrow C ABIs: buffers, `ZSTD_CCtx *` values, and job
//! descriptors remain opaque to Rust, while allocation, reuse, table-capacity
//! orchestration, dictionary branch/order policy, and sizing policy are
//! Rust-owned. C retains ownership of private descriptor fields and platform
//! synchronization.

use std::mem::{self, offset_of, size_of, MaybeUninit};
use std::os::raw::{c_int, c_uint, c_void};
use std::ptr;
use std::sync::Mutex;

use crate::bits::ZSTD_highbit32;
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::mem::MEM_writeLE32;
use crate::zstd_compress::{ZSTD_frameProgression, ZSTD_rust_invalidateRepCodes};
use crate::zstd_compress_frame::ZSTD_rust_writeFrameHeader;
use crate::zstd_compress_params::{
    ZSTD_compressionParameters, ZSTD_rust_params_updateCParamsWhileCompressing,
    ZSTD_rust_params_updateCParamsWhileCompressingState,
};

const ZSTDMT_JOBLOG_MAX: c_uint = if mem::size_of::<usize>() == 4 { 29 } else { 30 };
const ZSTD_WINDOWLOG_MAX: c_uint = if mem::size_of::<usize>() == 4 { 30 } else { 31 };
#[cfg(test)]
const DEFAULT_ZSTDMT_JOBSIZE_MIN: usize = 512 << 10;
#[cfg(test)]
const DEFAULT_ZSTDMT_JOBSIZE_MAX: usize = if mem::size_of::<usize>() == 4 {
    512 << 20
} else {
    1024 << 20
};

const ZSTD_FAST: c_int = 1;
const ZSTD_DFAST: c_int = 2;
const ZSTD_GREEDY: c_int = 3;
const ZSTD_LAZY: c_int = 4;
const ZSTD_LAZY2: c_int = 5;
const ZSTD_BTLAZY2: c_int = 6;
const ZSTD_BTOPT: c_int = 7;
const ZSTD_BTULTRA: c_int = 8;
const ZSTD_BTULTRA2: c_int = 9;
const ZSTD_PS_ENABLE: c_int = 1;
const ZSTD_PS_DISABLE: c_int = 2;
const ZSTD_DCT_RAW_CONTENT: c_int = 1;

const RSYNC_LENGTH: usize = 32;
const RSYNC_MIN_BLOCK_LOG: usize = 17;
const RSYNC_MIN_BLOCK_SIZE: usize = 1 << RSYNC_MIN_BLOCK_LOG;
const PRIME8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463;
const ROLL_HASH_CHAR_OFFSET: u64 = 10;
const _: () = assert!(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH);

type ZstdMtCompressFn =
    unsafe extern "C" fn(*mut c_void, *mut c_void, usize, *const c_void, usize) -> usize;
pub type ZSTDMT_chunkProgressFn = unsafe extern "C" fn(*mut c_void, usize, usize);

#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_chunkProcessResult {
    pub error: usize,
    pub lastBlockSize: usize,
}

/// Per-job parameter policy applied before a worker initializes its private
/// compression context. The full `ZSTD_CCtx_params` remains C-owned; only the
/// three scalar fields changed by the MT worker policy cross the ABI.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobParameters {
    pub checksumFlag: c_int,
    pub ldmEnable: c_int,
    pub nbWorkers: c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_compressionJobParameters, checksumFlag) == 0);
    assert!(offset_of!(ZSTDMT_compressionJobParameters, ldmEnable) == size_of::<c_int>());
    assert!(offset_of!(ZSTDMT_compressionJobParameters, nbWorkers) == 2 * size_of::<c_int>());
    assert!(
        size_of::<ZSTDMT_compressionJobParameters>()
            == 2 * size_of::<c_int>() + size_of::<c_uint>()
    );
};

/// Scalar MT state used when compression parameters are updated between
/// streaming jobs.  The private `ZSTDMT_CCtx` and `ZSTD_CCtx_params` layouts
/// remain in C; Rust owns the transition policy and returns the two values C
/// must publish together.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_CParamsUpdateProjection {
    pub compressionLevel: c_int,
    pub cctxSrcSizeHint: c_int,
    pub srcSizeHint: u64,
    pub dictSize: usize,
    pub mode: c_int,
    pub enableLdm: c_int,
    pub ldmDefaultWindowLog: c_uint,
    pub overrides: ZSTD_compressionParameters,
    pub useRowMatchFinder: c_int,
    pub exclusionMask: c_uint,
    pub savedWindowLog: c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_CParamsUpdateProjection, compressionLevel) == 0);
    assert!(offset_of!(ZSTDMT_CParamsUpdateProjection, cctxSrcSizeHint) == size_of::<c_int>());
    assert!(offset_of!(ZSTDMT_CParamsUpdateProjection, srcSizeHint) == 2 * size_of::<c_int>());
    assert!(offset_of!(ZSTDMT_CParamsUpdateProjection, dictSize) == 16);
    assert!(
        offset_of!(ZSTDMT_CParamsUpdateProjection, mode)
            == if size_of::<usize>() == 8 { 24 } else { 20 }
    );
    assert!(
        offset_of!(ZSTDMT_CParamsUpdateProjection, enableLdm)
            == if size_of::<usize>() == 8 { 28 } else { 24 }
    );
    assert!(
        offset_of!(ZSTDMT_CParamsUpdateProjection, ldmDefaultWindowLog)
            == if size_of::<usize>() == 8 { 32 } else { 28 }
    );
    assert!(
        offset_of!(ZSTDMT_CParamsUpdateProjection, overrides)
            == if size_of::<usize>() == 8 { 36 } else { 32 }
    );
    assert!(
        offset_of!(ZSTDMT_CParamsUpdateProjection, useRowMatchFinder)
            == if size_of::<usize>() == 8 { 64 } else { 60 }
    );
    assert!(
        offset_of!(ZSTDMT_CParamsUpdateProjection, exclusionMask)
            == if size_of::<usize>() == 8 { 68 } else { 64 }
    );
    assert!(
        offset_of!(ZSTDMT_CParamsUpdateProjection, savedWindowLog)
            == if size_of::<usize>() == 8 { 72 } else { 68 }
    );
    assert!(
        size_of::<ZSTDMT_CParamsUpdateProjection>()
            == if size_of::<usize>() == 8 { 80 } else { 72 }
    );
};

/// Result of the MT parameter transition.  Keeping the requested level next
/// to the derived compression parameters makes the C publication explicit:
/// C retains the private context, while Rust decides the complete scalar
/// state for the next job.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_CParamsUpdateResult {
    pub compressionLevel: c_int,
    pub cParams: ZSTD_compressionParameters,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_CParamsUpdateResult, compressionLevel) == 0);
    assert!(offset_of!(ZSTDMT_CParamsUpdateResult, cParams) == size_of::<c_int>());
    assert!(
        size_of::<ZSTDMT_CParamsUpdateResult>()
            == size_of::<c_int>() + size_of::<ZSTD_compressionParameters>()
    );
};

#[inline]
fn update_cparams_while_compressing(
    projection: ZSTDMT_CParamsUpdateProjection,
) -> ZSTDMT_CParamsUpdateResult {
    let c_params = ZSTD_rust_params_updateCParamsWhileCompressing(
        ZSTD_rust_params_updateCParamsWhileCompressingState {
            compressionLevel: projection.compressionLevel,
            cctxSrcSizeHint: projection.cctxSrcSizeHint,
            srcSizeHint: projection.srcSizeHint,
            dictSize: projection.dictSize,
            mode: projection.mode,
            enableLdm: projection.enableLdm,
            ldmDefaultWindowLog: projection.ldmDefaultWindowLog,
            overrides: projection.overrides,
            useRowMatchFinder: projection.useRowMatchFinder,
            exclusionMask: projection.exclusionMask,
            savedWindowLog: projection.savedWindowLog,
        },
    );
    ZSTDMT_CParamsUpdateResult {
        compressionLevel: projection.compressionLevel,
        cParams: c_params,
    }
}

/// Derive and publish the scalar MT parameter transition for the next job.
/// C keeps the private parameter object and performs the two final writes;
/// Rust owns the rule that the requested level travels with the derived
/// parameters and that the active frame window log is preserved.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_updateCParamsWhileCompressing(
    projection: ZSTDMT_CParamsUpdateProjection,
) -> ZSTDMT_CParamsUpdateResult {
    update_cparams_while_compressing(projection)
}

/// Scalar inputs for worker-resource acquisition. The worker descriptor,
/// synchronization, pools, and destination storage remain private to C;
/// Rust owns the acquisition/publication order and error short-circuiting.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobResourceProjection {
    pub ldmEnabled: c_int,
    pub hasDestination: c_int,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_compressionJobResourceProjection, ldmEnabled) == 0);
    assert!(
        offset_of!(ZSTDMT_compressionJobResourceProjection, hasDestination) == size_of::<c_int>()
    );
    assert!(size_of::<ZSTDMT_compressionJobResourceProjection>() == 2 * size_of::<c_int>());
};

pub type ZSTDMT_compressionJobResourceVoidFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_compressionJobResourceStepFn = unsafe extern "C" fn(*mut c_void) -> usize;
pub type ZSTDMT_compressionJobResourceReadyFn = unsafe extern "C" fn(*mut c_void) -> c_int;

#[inline]
fn prepare_compression_job_parameters(
    job_id: c_uint,
    checksum_flag: c_int,
    _ldm_enable: c_int,
    _nb_workers: c_uint,
) -> ZSTDMT_compressionJobParameters {
    ZSTDMT_compressionJobParameters {
        /* The first job writes the frame checksum; later jobs have their
         * checksum accounted for by the serial MT state. */
        checksumFlag: if job_id == 0 { checksum_flag } else { 0 },
        /* LDM sequences are generated by the serial state, not per worker. */
        ldmEnable: ZSTD_PS_DISABLE,
        /* Worker contexts are initialized as single-threaded contexts. */
        nbWorkers: 0,
    }
}

/// Apply the pure per-worker MT parameter policy while C retains the complete
/// compression-parameter structure and all codec initialization.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_prepareCompressionJobParameters(
    jobID: c_uint,
    checksumFlag: c_int,
    ldmEnable: c_int,
    nbWorkers: c_uint,
) -> ZSTDMT_compressionJobParameters {
    prepare_compression_job_parameters(jobID, checksumFlag, ldmEnable, nbWorkers)
}

#[inline]
fn compression_job_acquire_resources_with<A, S, C, B, R, P>(
    projection: ZSTDMT_compressionJobResourceProjection,
    mut acquire_cctx: A,
    mut acquire_raw_seq_store: S,
    mut has_cctx: C,
    mut acquire_destination_buffer: B,
    mut has_raw_seq_store: R,
    mut publish_destination: P,
) -> usize
where
    A: FnMut(),
    S: FnMut(),
    C: FnMut() -> bool,
    B: FnMut() -> usize,
    R: FnMut() -> bool,
    P: FnMut(),
{
    /* Keep both pool gets before checking the CCtx, matching the C worker
     * cleanup path that releases every resource acquired for the job. */
    acquire_cctx();
    acquire_raw_seq_store();
    if !has_cctx() {
        return ERROR(ZstdErrorCode::MemoryAllocation);
    }

    if projection.hasDestination == 0 {
        let error = acquire_destination_buffer();
        if ERR_isError(error) {
            return error;
        }
    }
    publish_destination();

    if projection.ldmEnabled != 0 && !has_raw_seq_store() {
        return ERROR(ZstdErrorCode::MemoryAllocation);
    }
    0
}

/// Own worker-resource acquisition policy while C retains pool, descriptor,
/// synchronization, and allocator operations behind callbacks. The order is
/// intentionally: acquire both worker resources, validate the CCtx, acquire a
/// destination only when absent, publish its scalar destination projection,
/// then validate the LDM sequence store.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressionJobAcquireResources(
    projection: *const ZSTDMT_compressionJobResourceProjection,
    opaque: *mut c_void,
    acquire_cctx: Option<ZSTDMT_compressionJobResourceVoidFn>,
    acquire_raw_seq_store: Option<ZSTDMT_compressionJobResourceVoidFn>,
    has_cctx: Option<ZSTDMT_compressionJobResourceReadyFn>,
    acquire_destination_buffer: Option<ZSTDMT_compressionJobResourceStepFn>,
    has_raw_seq_store: Option<ZSTDMT_compressionJobResourceReadyFn>,
    publish_destination: Option<ZSTDMT_compressionJobResourceVoidFn>,
) -> usize {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ERROR(ZstdErrorCode::Generic);
    };
    let (
        Some(acquire_cctx),
        Some(acquire_raw_seq_store),
        Some(has_cctx),
        Some(acquire_destination_buffer),
        Some(has_raw_seq_store),
        Some(publish_destination),
    ) = (
        acquire_cctx,
        acquire_raw_seq_store,
        has_cctx,
        acquire_destination_buffer,
        has_raw_seq_store,
        publish_destination,
    )
    else {
        return ERROR(ZstdErrorCode::Generic);
    };
    if opaque.is_null() {
        return ERROR(ZstdErrorCode::Generic);
    }

    unsafe {
        compression_job_acquire_resources_with(
            projection,
            || acquire_cctx(opaque),
            || acquire_raw_seq_store(opaque),
            || has_cctx(opaque) != 0,
            || acquire_destination_buffer(opaque),
            || has_raw_seq_store(opaque) != 0,
            || publish_destination(opaque),
        )
    }
}

/// Scalar job state used by the Rust compression-job scheduler.
///
/// The job descriptor, pools, synchronization, and codec state remain
/// private to C. Rust uses the frame-position flags and a scalar header
/// projection to choose the non-first-job header stage.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobProjection {
    pub firstJob: c_uint,
    pub lastJob: c_uint,
    pub frameHeaderState: *const ZSTDMT_compressionJobFrameHeaderState,
    pub sequenceState: *const ZSTDMT_compressionJobSequenceState,
}

/// Indirection into the worker-owned state used by the external-sequence
/// gate. Rust sees only pointers to the opaque CCtx and the already-projected
/// raw-sequence store; the private worker-state layout remains in C.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobSequenceState {
    pub cctx: *mut *mut c_void,
    pub rawSeqStore: *const ZstdMtRawSeqStore,
    pub ldmEnabled: c_int,
}

/// Scalar projection for the non-first MT job frame header. The destination,
/// stage, and repcodes point at the worker-owned C storage; all header
/// serialization and repcode invalidation happen in Rust.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobFrameHeaderState {
    dst: *mut c_void,
    dst_capacity: usize,
    stage: *mut c_int,
    no_dict_id_flag: c_int,
    checksum_flag: c_int,
    content_size_flag: c_int,
    format: c_int,
    window_log: c_uint,
    pledged_src_size_plus_one: u64,
    dict_id: c_uint,
    rep_codes: *mut c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_compressionJobProjection, firstJob) == 0);
    assert!(offset_of!(ZSTDMT_compressionJobProjection, lastJob) == size_of::<c_uint>());
    assert!(
        offset_of!(ZSTDMT_compressionJobProjection, frameHeaderState) == size_of::<[c_uint; 2]>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobProjection, sequenceState)
            == size_of::<[c_uint; 2]>() + size_of::<usize>()
    );
    assert!(
        size_of::<ZSTDMT_compressionJobProjection>()
            == size_of::<[c_uint; 2]>() + 2 * size_of::<usize>()
    );
    assert!(offset_of!(ZSTDMT_compressionJobSequenceState, cctx) == 0);
    assert!(
        offset_of!(ZSTDMT_compressionJobSequenceState, rawSeqStore) == size_of::<usize>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobSequenceState, ldmEnabled) == 2 * size_of::<usize>()
    );
    assert!(
        size_of::<ZSTDMT_compressionJobSequenceState>()
            == if size_of::<usize>() == 8 { 24 } else { 12 }
    );
    assert!(offset_of!(ZSTDMT_compressionJobFrameHeaderState, dst) == 0);
    assert!(offset_of!(ZSTDMT_compressionJobFrameHeaderState, dst_capacity) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressionJobFrameHeaderState, stage) == 2 * size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_compressionJobFrameHeaderState, no_dict_id_flag)
            == 3 * size_of::<usize>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFrameHeaderState, checksum_flag)
            == 3 * size_of::<usize>() + size_of::<c_int>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFrameHeaderState, content_size_flag)
            == 3 * size_of::<usize>() + 2 * size_of::<c_int>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFrameHeaderState, format)
            == 3 * size_of::<usize>() + 3 * size_of::<c_int>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFrameHeaderState, window_log)
            == 3 * size_of::<usize>() + 4 * size_of::<c_int>()
    );
    assert!(
        offset_of!(
            ZSTDMT_compressionJobFrameHeaderState,
            pledged_src_size_plus_one
        ) == ZSTDMT_HEADER_PLEDGED_OFFSET
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFrameHeaderState, dict_id)
            == ZSTDMT_HEADER_PLEDGED_OFFSET + size_of::<u64>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFrameHeaderState, rep_codes)
            == ZSTDMT_HEADER_REP_CODES_OFFSET
    );
    assert!(
        size_of::<ZSTDMT_compressionJobFrameHeaderState>()
            == if size_of::<usize>() == 8 { 72 } else { 48 }
    );
};

const ZSTD_COMPRESSION_STAGE_CREATED: c_int = 0;
const ZSTD_COMPRESSION_STAGE_INIT: c_int = 1;
const ZSTD_COMPRESSION_STAGE_ONGOING: c_int = 2;
const ZSTDMT_HEADER_PLEDGED_OFFSET: usize = if size_of::<usize>() == 8 {
    size_of::<[usize; 6]>()
} else {
    size_of::<[usize; 8]>()
};
const ZSTDMT_HEADER_REP_CODES_OFFSET: usize = if size_of::<usize>() == 8 {
    size_of::<[usize; 8]>()
} else {
    size_of::<[usize; 11]>()
};

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_writeFrameHeader(
    state: *const ZSTDMT_compressionJobFrameHeaderState,
) -> usize {
    if state.is_null() {
        return ERROR(ZstdErrorCode::Generic);
    }
    let state = unsafe { &*state };
    if state.stage.is_null() || state.rep_codes.is_null() {
        return ERROR(ZstdErrorCode::Generic);
    }
    if unsafe { *state.stage } == ZSTD_COMPRESSION_STAGE_CREATED {
        return ERROR(ZstdErrorCode::StageWrong);
    }

    let header_size = if unsafe { *state.stage } == ZSTD_COMPRESSION_STAGE_INIT {
        let header_size = unsafe {
            ZSTD_rust_writeFrameHeader(
                state.dst,
                state.dst_capacity,
                state.no_dict_id_flag,
                state.checksum_flag,
                state.content_size_flag,
                state.format,
                state.window_log,
                state.pledged_src_size_plus_one.wrapping_sub(1),
                state.dict_id,
            )
        };
        if ERR_isError(header_size) {
            return header_size;
        }
        if header_size > state.dst_capacity {
            return ERROR(ZstdErrorCode::DstSizeTooSmall);
        }
        unsafe { *state.stage = ZSTD_COMPRESSION_STAGE_ONGOING };
        header_size
    } else {
        0
    };

    unsafe { ZSTD_rust_invalidateRepCodes(state.rep_codes) };
    header_size
}

pub type ZSTDMT_compressionJobStepFn = unsafe extern "C" fn(*mut c_void) -> usize;
pub type ZSTDMT_compressionJobVoidFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_compressionJobCompressFn =
    unsafe extern "C" fn(*mut c_void, c_uint) -> ZSTDMT_chunkProcessResult;
pub type ZSTDMT_compressionJobSizeFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_compressionJobSetParameterFn = unsafe extern "C" fn(*mut c_void, c_int) -> usize;
pub type ZSTDMT_compressionJobApplySequencesFn =
    unsafe extern "C" fn(*mut c_void, *mut c_void, *mut c_void, usize);

/// Live fields and synchronization callbacks for one worker's intermediate
/// compression progress.  The private job descriptor and pthread objects
/// remain in C; Rust owns the publication order and wrapping counter update.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_compressionJobProgressProjection {
    pub callbackContext: *mut c_void,
    pub cSize: *mut usize,
    pub consumed: *mut usize,
    pub lock: Option<ZSTDMT_compressionJobVoidFn>,
    pub signal: Option<ZSTDMT_compressionJobVoidFn>,
    pub unlock: Option<ZSTDMT_compressionJobVoidFn>,
    pub debug: Option<ZSTDMT_chunkProgressFn>,
}

const _: () = {
    assert!(
        offset_of!(ZSTDMT_compressionJobProgressProjection, callbackContext) == 0
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobProgressProjection, cSize) == size_of::<*mut c_void>()
    );
    assert!(offset_of!(ZSTDMT_compressionJobProgressProjection, consumed)
        == size_of::<*mut c_void>() + size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressionJobProgressProjection, lock)
        == size_of::<*mut c_void>() + 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressionJobProgressProjection, signal)
        == size_of::<*mut c_void>() + 2 * size_of::<usize>() + size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressionJobProgressProjection, unlock)
        == size_of::<*mut c_void>() + 2 * size_of::<usize>() + 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressionJobProgressProjection, debug)
        == size_of::<*mut c_void>() + 2 * size_of::<usize>() + 3 * size_of::<usize>());
    assert!(size_of::<ZSTDMT_compressionJobProgressProjection>()
        == size_of::<*mut c_void>() + 2 * size_of::<usize>() + 4 * size_of::<usize>());
};

/// Publish one intermediate worker block while preserving the C callback's
/// lock, update, diagnostic, signal, and unlock order.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressionJobProgress(
    opaque: *mut c_void,
    c_size: usize,
    consumed: usize,
) {
    let Some(state) = (unsafe {
        opaque
            .cast::<ZSTDMT_compressionJobProgressProjection>()
            .as_ref()
    }) else {
        return;
    };
    let (Some(lock), Some(signal), Some(unlock)) = (state.lock, state.signal, state.unlock)
    else {
        return;
    };
    if state.cSize.is_null() || state.consumed.is_null() {
        return;
    }

    unsafe {
        lock(state.callbackContext);
        *state.cSize = (*state.cSize).wrapping_add(c_size);
        *state.consumed = consumed;
        if let Some(debug) = state.debug {
            debug(state.callbackContext, c_size, *state.cSize);
        }
        signal(state.callbackContext);
        unlock(state.callbackContext);
    }
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct ZSTDMT_compressionJobSequenceProjection {
    cctx: *mut c_void,
    sequences: *mut c_void,
    nbSequences: usize,
    ldmEnabled: c_int,
}

#[inline]
#[cfg(not(test))]
unsafe fn compression_job_sequence_projection(
    state: *const ZSTDMT_compressionJobSequenceState,
) -> Option<ZSTDMT_compressionJobSequenceProjection> {
    let state = unsafe { state.as_ref() }?;
    if state.cctx.is_null() || state.rawSeqStore.is_null() {
        return None;
    }

    let cctx = unsafe { *state.cctx };
    let raw_seq_store = unsafe { &*state.rawSeqStore };
    Some(ZSTDMT_compressionJobSequenceProjection {
        cctx,
        sequences: raw_seq_store.seq.cast(),
        nbSequences: raw_seq_store.size,
        ldmEnabled: state.ldmEnabled,
    })
}

/// Apply the worker's external-sequence policy after CCtx initialization.
/// Rust owns the non-empty gate and LDM invariant; C retains only the opaque
/// `ZSTD_referenceExternalSequences()` leaf behind the callback.
#[inline]
fn apply_compression_job_sequences_with<R>(
    projection: Option<ZSTDMT_compressionJobSequenceProjection>,
    mut reference_sequences: R,
) where
    R: FnMut(*mut c_void, *mut c_void, usize),
{
    let Some(projection) = projection else {
        return;
    };
    if projection.nbSequences == 0 {
        return;
    }

    debug_assert_eq!(projection.ldmEnabled, ZSTD_PS_ENABLE);
    debug_assert!(!projection.cctx.is_null());
    reference_sequences(projection.cctx, projection.sequences, projection.nbSequences);
}

/// Scalar view used to choose the private worker-context initialization path.
/// C retains the cdict, parameter, and CCtx objects; Rust owns the branch and
/// the rule that the frame-header projection is published only after init.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressionJobBeginProjection {
    pub firstJob: c_uint,
    pub hasCDict: c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_compressionJobBeginProjection, firstJob) == 0);
    assert!(offset_of!(ZSTDMT_compressionJobBeginProjection, hasCDict) == size_of::<c_uint>());
    assert!(size_of::<ZSTDMT_compressionJobBeginProjection>() == 2 * size_of::<c_uint>());
};

#[inline]
fn begin_compression_job_with<B, F, D, P, H>(
    projection: ZSTDMT_compressionJobBeginProjection,
    mut begin_with_cdict: B,
    mut set_force_max_window: F,
    mut set_deterministic_ref_prefix: D,
    mut begin_with_prefix: P,
    mut publish_frame_header: H,
) -> usize
where
    B: FnMut() -> usize,
    F: FnMut(c_int) -> usize,
    D: FnMut() -> usize,
    P: FnMut() -> usize,
    H: FnMut(),
{
    let init_error = if projection.hasCDict != 0 {
        if projection.firstJob == 0 {
            return ERROR(ZstdErrorCode::StageWrong);
        }
        begin_with_cdict()
    } else {
        let force_window_error = set_force_max_window((projection.firstJob == 0) as c_int);
        if ERR_isError(force_window_error) {
            return force_window_error;
        }
        if projection.firstJob == 0 {
            let error = set_deterministic_ref_prefix();
            if ERR_isError(error) {
                return error;
            }
        }
        begin_with_prefix()
    };

    if ERR_isError(init_error) {
        return init_error;
    }
    publish_frame_header();
    init_error
}

/// Own the worker-context initialization branch and failure/publication order
/// while C keeps all private codec operations behind callbacks.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressionJobBegin(
    projection: *const ZSTDMT_compressionJobBeginProjection,
    opaque: *mut c_void,
    beginWithCDict: Option<ZSTDMT_compressionJobStepFn>,
    setForceMaxWindow: Option<ZSTDMT_compressionJobSetParameterFn>,
    setDeterministicRefPrefix: Option<ZSTDMT_compressionJobStepFn>,
    beginWithPrefix: Option<ZSTDMT_compressionJobStepFn>,
    publishFrameHeader: Option<ZSTDMT_compressionJobVoidFn>,
) -> usize {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ERROR(ZstdErrorCode::Generic);
    };
    let (
        Some(begin_with_cdict),
        Some(set_force_max_window),
        Some(set_deterministic_ref_prefix),
        Some(begin_with_prefix),
        Some(publish_frame_header),
    ) = (
        beginWithCDict,
        setForceMaxWindow,
        setDeterministicRefPrefix,
        beginWithPrefix,
        publishFrameHeader,
    )
    else {
        return ERROR(ZstdErrorCode::Generic);
    };

    begin_compression_job_with(
        projection,
        || unsafe { begin_with_cdict(opaque) },
        |value| unsafe { set_force_max_window(opaque, value) },
        || unsafe { set_deterministic_ref_prefix(opaque) },
        || unsafe { begin_with_prefix(opaque) },
        || unsafe { publish_frame_header(opaque) },
    )
}

/// Live fields and synchronization callbacks for worker-job finish
/// publication. The private job descriptor and pthread objects remain in C;
/// Rust owns the counter updates and completion signal order.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_compressionJobFinishPublicationProjection {
    pub callbackContext: *mut c_void,
    pub cSize: *mut usize,
    pub consumed: *mut usize,
    pub srcSize: usize,
    pub lock: Option<ZSTDMT_compressionJobVoidFn>,
    pub signal: Option<ZSTDMT_compressionJobVoidFn>,
    pub unlock: Option<ZSTDMT_compressionJobVoidFn>,
    pub validateConsumed: Option<ZSTDMT_compressionJobSizeFn>,
}

const _: () = {
    assert!(
        offset_of!(ZSTDMT_compressionJobFinishPublicationProjection, callbackContext) == 0
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFinishPublicationProjection, cSize)
            == size_of::<*mut c_void>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFinishPublicationProjection, consumed)
            == size_of::<*mut c_void>() + size_of::<usize>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFinishPublicationProjection, srcSize)
            == size_of::<*mut c_void>() + 2 * size_of::<usize>()
    );
    assert!(offset_of!(ZSTDMT_compressionJobFinishPublicationProjection, lock)
        == size_of::<*mut c_void>() + 3 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressionJobFinishPublicationProjection, signal)
        == size_of::<*mut c_void>() + 3 * size_of::<usize>() + size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressionJobFinishPublicationProjection, unlock)
        == size_of::<*mut c_void>() + 3 * size_of::<usize>() + 2 * size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_compressionJobFinishPublicationProjection, validateConsumed)
            == size_of::<*mut c_void>() + 3 * size_of::<usize>() + 3 * size_of::<usize>()
    );
    assert!(size_of::<ZSTDMT_compressionJobFinishPublicationProjection>()
        == size_of::<*mut c_void>() + 3 * size_of::<usize>() + 4 * size_of::<usize>());
};

/// C-owned leaves for the worker-job finish path.
///
/// Rust owns the error/success branch and callback order. C keeps the job
/// descriptor, pthread objects, private pools, and serial state behind the
/// remaining callbacks; publication state crosses through one live projection.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_compressionJobFinishProjection {
    pub callbackContext: *mut c_void,
    pub srcSize: usize,
    pub ensureFinished: Option<ZSTDMT_compressionJobVoidFn>,
    pub publication: ZSTDMT_compressionJobFinishPublicationProjection,
    pub releaseSeq: Option<ZSTDMT_compressionJobVoidFn>,
    pub releaseCCtx: Option<ZSTDMT_compressionJobVoidFn>,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_compressionJobFinishProjection, callbackContext) == 0);
    assert!(offset_of!(ZSTDMT_compressionJobFinishProjection, srcSize) == size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_compressionJobFinishProjection, ensureFinished) == 2 * size_of::<usize>()
    );
    assert!(offset_of!(ZSTDMT_compressionJobFinishProjection, publication)
        == 3 * size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_compressionJobFinishProjection, releaseSeq)
            == 3 * size_of::<usize>()
                + size_of::<ZSTDMT_compressionJobFinishPublicationProjection>()
    );
    assert!(
        offset_of!(ZSTDMT_compressionJobFinishProjection, releaseCCtx)
            == 4 * size_of::<usize>()
                + size_of::<ZSTDMT_compressionJobFinishPublicationProjection>()
    );
    assert!(size_of::<ZSTDMT_compressionJobFinishProjection>()
        == 5 * size_of::<usize>()
            + size_of::<ZSTDMT_compressionJobFinishPublicationProjection>());
};

/// C retains the private window assertion and trace-recording leaves for an
/// MT worker. Rust owns the first-job policy that determines whether the
/// external-dictionary assertion is required before tracing.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_compressionJobTraceProjection {
    pub callbackContext: *mut c_void,
    pub firstJob: c_uint,
    pub assertNoExtDict: Option<ZSTDMT_compressionJobVoidFn>,
    pub trace: Option<ZSTDMT_compressionJobVoidFn>,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_compressionJobTraceProjection, callbackContext) == 0);
    assert!(offset_of!(ZSTDMT_compressionJobTraceProjection, firstJob) == size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_compressionJobTraceProjection, assertNoExtDict)
            == 2 * size_of::<usize>()
    );
    assert!(offset_of!(ZSTDMT_compressionJobTraceProjection, trace) == 3 * size_of::<usize>());
    assert!(size_of::<ZSTDMT_compressionJobTraceProjection>() == 4 * size_of::<usize>());
};

#[inline]
fn compression_job_trace_with<A, T>(first_job: c_uint, mut assert_no_ext_dict: A, mut trace: T)
where
    A: FnMut(),
    T: FnMut(),
{
    if first_job == 0 {
        assert_no_ext_dict();
    }
    trace();
}

/// Apply the worker trace policy while C retains the opaque trace leaves.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressionJobTrace(
    projection: *const ZSTDMT_compressionJobTraceProjection,
) {
    let Some(projection) = (unsafe { projection.as_ref() }) else {
        return;
    };
    let Some(trace) = projection.trace else {
        return;
    };
    if projection.firstJob == 0 && projection.assertNoExtDict.is_none() {
        return;
    }
    let assert_no_ext_dict = projection.assertNoExtDict;
    compression_job_trace_with(
        projection.firstJob,
        || {
            if let Some(assert_no_ext_dict) = assert_no_ext_dict {
                unsafe { assert_no_ext_dict(projection.callbackContext) };
            }
        },
        || unsafe { trace(projection.callbackContext) },
    );
}

pub type ZSTDMT_serialWaitForTurnFn = unsafe extern "C" fn(*mut c_void, c_uint) -> c_int;
pub type ZSTDMT_serialLdmWindowUpdateFn =
    unsafe extern "C" fn(*mut c_void, *mut ZstdMtRawSeqStore, *const c_void, usize);
pub type ZSTDMT_serialLdmGenerateSequencesFn =
    unsafe extern "C" fn(*mut c_void, *mut ZstdMtRawSeqStore, *const c_void, usize);
pub type ZSTDMT_serialLdmPublishWindowFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialUpdateChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize);
pub type ZSTDMT_serialAdvanceFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateLockFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateWaitFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateBroadcastFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateUnlockFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateCallbackFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateSkipFn = unsafe extern "C" fn(*mut c_void, c_uint, usize);

/// Projection for the MT serial turn wait.  Rust owns the lock/wait loop and
/// comparison; C retains the pthread mutex and condition variable behind the
/// callbacks.  The lock intentionally remains held when this returns so the
/// caller can perform the serial turn and release it in the advance callback.
#[repr(C)]
pub struct ZSTDMT_RustSerialWaitForTurnState {
    callback_context: *mut c_void,
    next_job_id: *mut c_uint,
    lock: Option<ZSTDMT_serialStateLockFn>,
    wait: Option<ZSTDMT_serialStateWaitFn>,
    job_id: c_uint,
}

const _: () = {
    assert!(size_of::<ZSTDMT_serialStateLockFn>() == size_of::<usize>());
    assert!(size_of::<ZSTDMT_serialStateWaitFn>() == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, callback_context) == 0);
    assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, next_job_id) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, lock) == 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, wait) == 3 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialWaitForTurnState, job_id) == 4 * size_of::<usize>());
    assert!(size_of::<ZSTDMT_RustSerialWaitForTurnState>() == size_of::<[usize; 5]>());
};

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateWaitForTurn(
    state: *const ZSTDMT_RustSerialWaitForTurnState,
) -> c_int {
    if state.is_null() {
        return 0;
    }
    let state = unsafe { &*state };
    if state.callback_context.is_null() || state.next_job_id.is_null() {
        return 0;
    }
    let (Some(lock), Some(wait)) = (state.lock, state.wait) else {
        return 0;
    };

    unsafe {
        lock(state.callback_context);
        loop {
            if *state.next_job_id >= state.job_id {
                break;
            }
            wait(state.callback_context);
        }
        c_int::from(*state.next_job_id == state.job_id)
    }
}

/// Projection for advancing the MT serial turn.  Rust owns the increment and
/// callback order; C retains the condition variable broadcast and mutex
/// unlock operations behind the callbacks.
#[repr(C)]
pub struct ZSTDMT_RustSerialAdvanceState {
    callback_context: *mut c_void,
    next_job_id: *mut c_uint,
    broadcast: Option<ZSTDMT_serialStateBroadcastFn>,
    unlock: Option<ZSTDMT_serialStateUnlockFn>,
}

const _: () = {
    assert!(size_of::<ZSTDMT_serialStateBroadcastFn>() == size_of::<usize>());
    assert!(size_of::<ZSTDMT_serialStateUnlockFn>() == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialAdvanceState, callback_context) == 0);
    assert!(offset_of!(ZSTDMT_RustSerialAdvanceState, next_job_id) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialAdvanceState, broadcast) == 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialAdvanceState, unlock) == 3 * size_of::<usize>());
    assert!(size_of::<ZSTDMT_RustSerialAdvanceState>() == size_of::<[usize; 4]>());
};

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateAdvance(
    state: *const ZSTDMT_RustSerialAdvanceState,
) {
    if state.is_null() {
        return;
    }
    let state = unsafe { &*state };
    if state.callback_context.is_null() || state.next_job_id.is_null() {
        return;
    }
    let (Some(broadcast), Some(unlock)) = (state.broadcast, state.unlock) else {
        return;
    };

    unsafe {
        *state.next_job_id = (*state.next_job_id).wrapping_add(1);
        broadcast(state.callback_context);
        unlock(state.callback_context);
    }
}

/// Projection for the failed-job serial cleanup path.  Rust owns the lock,
/// skip decision, serial-counter publication, and callback ordering; C keeps
/// the mutex, condition variables, and private LDM-window cleanup behind the
/// callbacks.
#[repr(C)]
pub struct ZSTDMT_RustSerialEnsureFinishedState {
    callback_context: *mut c_void,
    next_job_id: *mut c_uint,
    lock: Option<ZSTDMT_serialStateCallbackFn>,
    broadcast: Option<ZSTDMT_serialStateCallbackFn>,
    ldm_lock: Option<ZSTDMT_serialStateCallbackFn>,
    clear_ldm_window: Option<ZSTDMT_serialStateCallbackFn>,
    ldm_signal: Option<ZSTDMT_serialStateCallbackFn>,
    ldm_unlock: Option<ZSTDMT_serialStateCallbackFn>,
    unlock: Option<ZSTDMT_serialStateCallbackFn>,
    on_skip: Option<ZSTDMT_serialStateSkipFn>,
    c_size: usize,
    job_id: c_uint,
}

const _: () = {
    assert!(size_of::<ZSTDMT_serialStateCallbackFn>() == size_of::<usize>());
    assert!(size_of::<ZSTDMT_serialStateSkipFn>() == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, callback_context) == 0);
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, next_job_id) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, lock) == 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, broadcast) == 3 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, ldm_lock) == 4 * size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_RustSerialEnsureFinishedState, clear_ldm_window)
            == 5 * size_of::<usize>()
    );
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, ldm_signal) == 6 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, ldm_unlock) == 7 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, unlock) == size_of::<[usize; 8]>());
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, on_skip) == 9 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustSerialEnsureFinishedState, c_size) == 10 * size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_RustSerialEnsureFinishedState, job_id)
            == 10 * size_of::<usize>() + size_of::<usize>()
    );
    assert!(size_of::<ZSTDMT_RustSerialEnsureFinishedState>() == size_of::<[usize; 12]>());
};

#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_serialStateEnsureFinishedResult {
    pub skip: c_uint,
    pub nextJobID: c_uint,
}
pub type ZSTDMT_waitForJobCompleteFn = unsafe extern "C" fn(*mut c_void, c_uint, c_uint);
pub type ZSTDMT_releaseJobResourceFn = unsafe extern "C" fn(*mut c_void, c_uint);

pub type ZSTDMT_freeCCtxFn = unsafe extern "C" fn(*mut c_void);

/// Projection for MT context teardown.  The context and every allocation or
/// synchronization object remain private to C; Rust owns only the ownership
/// branches and teardown ordering between callbacks.
#[repr(C)]
pub struct ZSTDMT_RustFreeCCtxState {
    callback_context: *mut c_void,
    provided_factory: usize,
    round_buffer_present: usize,
    free_factory: Option<ZSTDMT_freeCCtxFn>,
    release_all_job_resources: Option<ZSTDMT_freeCCtxFn>,
    free_jobs: Option<ZSTDMT_freeCCtxFn>,
    free_buffer_pool: Option<ZSTDMT_freeCCtxFn>,
    free_cctx_pool: Option<ZSTDMT_freeCCtxFn>,
    free_seq_pool: Option<ZSTDMT_freeCCtxFn>,
    free_serial_state: Option<ZSTDMT_freeCCtxFn>,
    free_cdict: Option<ZSTDMT_freeCCtxFn>,
    free_round_buffer: Option<ZSTDMT_freeCCtxFn>,
    free_mtctx: Option<ZSTDMT_freeCCtxFn>,
}

const _: () = {
    assert!(size_of::<ZSTDMT_freeCCtxFn>() == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, callback_context) == 0);
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, provided_factory) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, round_buffer_present) == 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_factory) == 3 * size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_RustFreeCCtxState, release_all_job_resources) == 4 * size_of::<usize>()
    );
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_jobs) == 5 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_buffer_pool) == 6 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_cctx_pool) == 7 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_seq_pool) == 8 * (usize::BITS as usize / 8));
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_serial_state) == 9 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_cdict) == 10 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_round_buffer) == 11 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeCCtxState, free_mtctx) == 12 * size_of::<usize>());
    assert!(size_of::<ZSTDMT_RustFreeCCtxState>() == size_of::<[usize; 13]>());
};

pub type ZSTDMT_createCCtxAllocateFn = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
pub type ZSTDMT_createCCtxSetWorkersFn = unsafe extern "C" fn(*mut c_void, c_uint) -> usize;
pub type ZSTDMT_createCCtxSetInitialStateFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_createCCtxFactoryFn = unsafe extern "C" fn(*mut c_void, c_uint) -> *mut c_void;
pub type ZSTDMT_createCCtxJobsFn = unsafe extern "C" fn(*mut c_void, *mut c_uint) -> *mut c_void;
pub type ZSTDMT_createCCtxResourceFn = unsafe extern "C" fn(*mut c_void, c_uint) -> *mut c_void;
pub type ZSTDMT_createCCtxSerialInitFn = unsafe extern "C" fn(*mut c_void) -> c_int;
pub type ZSTDMT_createCCtxFreeFn = unsafe extern "C" fn(*mut c_void);

/// Scalar inputs and callbacks for MT context construction.  Rust owns the
/// validation, worker clamp, and resource-construction order; C retains the
/// private context layout, allocator calls, pools, and synchronization.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_RustCreateCCtxProjection {
    callback_context: *mut c_void,
    requested_nb_workers: c_uint,
    max_nb_workers: c_uint,
    context_size: usize,
    custom_mem: ZstdCustomMem,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_RustCreateCCtxProjection, callback_context) == 0);
    assert!(
        offset_of!(ZSTDMT_RustCreateCCtxProjection, requested_nb_workers) == size_of::<usize>()
    );
    assert!(
        offset_of!(ZSTDMT_RustCreateCCtxProjection, max_nb_workers)
            == size_of::<usize>() + size_of::<c_uint>()
    );
    assert!(
        offset_of!(ZSTDMT_RustCreateCCtxProjection, context_size)
            == if size_of::<usize>() == 8 { 16 } else { 12 }
    );
    assert!(
        offset_of!(ZSTDMT_RustCreateCCtxProjection, custom_mem)
            == if size_of::<usize>() == 8 { 24 } else { 16 }
    );
    assert!(
        size_of::<ZSTDMT_RustCreateCCtxProjection>()
            == if size_of::<usize>() == 8 { 48 } else { 28 }
    );
};

fn custom_mem_is_valid(custom_mem: ZstdCustomMem) -> bool {
    custom_mem.customAlloc.is_some() == custom_mem.customFree.is_some()
}

/// Construct an MT context through the original C resource callbacks.
/// Callbacks are all invoked through the allocation and cleanup sequence even
/// after an intermediate resource returns NULL, matching the C constructor's
/// cleanup contract for partially initialized contexts.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_createCCtx(
    projection: *const ZSTDMT_RustCreateCCtxProjection,
    allocate: Option<ZSTDMT_createCCtxAllocateFn>,
    set_workers: Option<ZSTDMT_createCCtxSetWorkersFn>,
    set_initial_state: Option<ZSTDMT_createCCtxSetInitialStateFn>,
    create_factory: Option<ZSTDMT_createCCtxFactoryFn>,
    create_jobs: Option<ZSTDMT_createCCtxJobsFn>,
    create_buffer_pool: Option<ZSTDMT_createCCtxResourceFn>,
    create_cctx_pool: Option<ZSTDMT_createCCtxResourceFn>,
    create_seq_pool: Option<ZSTDMT_createCCtxResourceFn>,
    init_serial: Option<ZSTDMT_createCCtxSerialInitFn>,
    free_context: Option<ZSTDMT_createCCtxFreeFn>,
) -> *mut c_void {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ptr::null_mut();
    };
    let (
        Some(allocate),
        Some(set_workers),
        Some(set_initial_state),
        Some(create_factory),
        Some(create_jobs),
        Some(create_buffer_pool),
        Some(create_cctx_pool),
        Some(create_seq_pool),
        Some(init_serial),
        Some(free_context),
    ) = (
        allocate,
        set_workers,
        set_initial_state,
        create_factory,
        create_jobs,
        create_buffer_pool,
        create_cctx_pool,
        create_seq_pool,
        init_serial,
        free_context,
    )
    else {
        return ptr::null_mut();
    };

    if projection.callback_context.is_null()
        || projection.requested_nb_workers == 0
        || projection.max_nb_workers == 0
        || projection.context_size == 0
        || !custom_mem_is_valid(projection.custom_mem)
    {
        return ptr::null_mut();
    }

    let nb_workers = projection
        .requested_nb_workers
        .min(projection.max_nb_workers);
    let mut nb_jobs = projection.requested_nb_workers.wrapping_add(2);
    let context = unsafe { allocate(projection.callback_context, projection.context_size) };
    if context.is_null() {
        return ptr::null_mut();
    }

    // The C constructor ignores this setter's result for a valid worker
    // count; preserve that exact policy before publishing the custom memory.
    unsafe {
        let _ = set_workers(projection.callback_context, nb_workers);
        set_initial_state(projection.callback_context);
    }

    let factory = unsafe { create_factory(projection.callback_context, nb_workers) };
    let jobs = unsafe { create_jobs(projection.callback_context, &mut nb_jobs) };
    let buffer_pool = unsafe { create_buffer_pool(projection.callback_context, nb_workers) };
    let cctx_pool = unsafe { create_cctx_pool(projection.callback_context, nb_workers) };
    let seq_pool = unsafe { create_seq_pool(projection.callback_context, nb_workers) };
    let serial_error = unsafe { init_serial(projection.callback_context) };

    if factory.is_null()
        || jobs.is_null()
        || buffer_pool.is_null()
        || cctx_pool.is_null()
        || seq_pool.is_null()
        || serial_error != 0
    {
        unsafe { free_context(projection.callback_context) };
        return ptr::null_mut();
    }

    context
}

pub type ZSTDMT_resizeStepFn = unsafe extern "C" fn(*mut c_void, c_uint) -> usize;
pub type ZSTDMT_resizeResourceFn = unsafe extern "C" fn(*mut c_void, c_uint) -> *mut c_void;

/// Scalar inputs for MT pool resizing. Rust owns the resize order and
/// short-circuit policy; C retains every private pool and context mutation
/// behind these callbacks.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_RustResizeProjection {
    callback_context: *mut c_void,
    nb_workers: c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_RustResizeProjection, callback_context) == 0);
    assert!(offset_of!(ZSTDMT_RustResizeProjection, nb_workers) == size_of::<usize>());
    assert!(
        size_of::<ZSTDMT_RustResizeProjection>() == if size_of::<usize>() == 8 { 16 } else { 8 }
    );
};

/// Resize MT resources in the original order: worker factory, jobs table,
/// buffer pool, CCtx pool, sequence pool, then the parameter publication.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_resize(
    projection: *const ZSTDMT_RustResizeProjection,
    resize_factory: Option<ZSTDMT_resizeStepFn>,
    expand_jobs: Option<ZSTDMT_resizeStepFn>,
    expand_buffer_pool: Option<ZSTDMT_resizeResourceFn>,
    expand_cctx_pool: Option<ZSTDMT_resizeResourceFn>,
    expand_seq_pool: Option<ZSTDMT_resizeResourceFn>,
    set_nb_workers: Option<ZSTDMT_resizeStepFn>,
) -> usize {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ERROR(ZstdErrorCode::Generic);
    };
    let (
        Some(resize_factory),
        Some(expand_jobs),
        Some(expand_buffer_pool),
        Some(expand_cctx_pool),
        Some(expand_seq_pool),
        Some(set_nb_workers),
    ) = (
        resize_factory,
        expand_jobs,
        expand_buffer_pool,
        expand_cctx_pool,
        expand_seq_pool,
        set_nb_workers,
    )
    else {
        return ERROR(ZstdErrorCode::Generic);
    };
    if projection.callback_context.is_null() {
        return ERROR(ZstdErrorCode::Generic);
    }

    let context = projection.callback_context;
    let workers = projection.nb_workers;
    let error = unsafe { resize_factory(context, workers) };
    if ERR_isError(error) {
        return error;
    }
    let error = unsafe { expand_jobs(context, workers) };
    if ERR_isError(error) {
        return error;
    }
    if unsafe { expand_buffer_pool(context, workers) }.is_null() {
        return ERROR(ZstdErrorCode::MemoryAllocation);
    }
    if unsafe { expand_cctx_pool(context, workers) }.is_null() {
        return ERROR(ZstdErrorCode::MemoryAllocation);
    }
    if unsafe { expand_seq_pool(context, workers) }.is_null() {
        return ERROR(ZstdErrorCode::MemoryAllocation);
    }
    unsafe { set_nb_workers(context, workers) }
}

type ZSTDMT_waitForLdmLockFn = unsafe extern "C" fn(*mut c_void);
type ZSTDMT_waitForLdmOverlapFn = unsafe extern "C" fn(*mut c_void, *mut c_void, usize) -> c_int;
type ZSTDMT_waitForLdmWaitFn = unsafe extern "C" fn(*mut c_void);
type ZSTDMT_waitForLdmUnlockFn = unsafe extern "C" fn(*mut c_void);

/// Projection for the MT wait that protects reusable input against LDM use.
///
/// Rust owns the enabled branch and lock/overlap/wait ordering. C retains the
/// pthread mutex, condition variable, and private LDM window state.
#[repr(C)]
pub struct ZSTDMT_RustWaitForLdmState {
    callback_context: *mut c_void,
    buffer_start: *mut c_void,
    buffer_capacity: usize,
    ldm_enabled: c_int,
    lock: Option<ZSTDMT_waitForLdmLockFn>,
    overlaps: Option<ZSTDMT_waitForLdmOverlapFn>,
    wait: Option<ZSTDMT_waitForLdmWaitFn>,
    unlock: Option<ZSTDMT_waitForLdmUnlockFn>,
}

const _: () = {
    assert!(size_of::<ZSTDMT_waitForLdmLockFn>() == size_of::<usize>());
    assert!(size_of::<ZSTDMT_waitForLdmOverlapFn>() == size_of::<usize>());
    assert!(size_of::<ZSTDMT_waitForLdmWaitFn>() == size_of::<usize>());
    assert!(size_of::<ZSTDMT_waitForLdmUnlockFn>() == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustWaitForLdmState, callback_context) == 0);
    assert!(offset_of!(ZSTDMT_RustWaitForLdmState, buffer_start) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustWaitForLdmState, buffer_capacity) == 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustWaitForLdmState, ldm_enabled) == 3 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustWaitForLdmState, lock) == 4 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustWaitForLdmState, overlaps) == 5 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustWaitForLdmState, wait) == 6 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustWaitForLdmState, unlock) == 7 * size_of::<usize>());
    assert!(size_of::<ZSTDMT_RustWaitForLdmState>() == size_of::<[usize; 8]>());
};

/// Wait until the reusable input range no longer overlaps the serial LDM
/// window, preserving the C synchronization state behind callbacks.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_waitForLdmComplete(state: *const ZSTDMT_RustWaitForLdmState) {
    if state.is_null() {
        return;
    }
    let state = unsafe { &*state };
    if state.ldm_enabled == 0 || state.callback_context.is_null() {
        return;
    }
    let (Some(lock), Some(overlaps), Some(wait), Some(unlock)) =
        (state.lock, state.overlaps, state.wait, state.unlock)
    else {
        return;
    };

    unsafe {
        lock(state.callback_context);
        while overlaps(
            state.callback_context,
            state.buffer_start,
            state.buffer_capacity,
        ) != 0
        {
            wait(state.callback_context);
        }
        unlock(state.callback_context);
    }
}

/// Scalar inputs for the MT streaming initializer.  The full parameter
/// object, dictionary handles, pools, buffers, and synchronization remain
/// private to C. Rust owns the order in which the C callbacks are invoked and
/// the pure normalization/sizing decisions between those callbacks.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_initCStreamProjection {
    pub requestedNbWorkers: c_uint,
    pub currentNbWorkers: c_uint,
    pub jobSize: usize,
    pub jobSizeMin: usize,
    pub jobSizeMax: usize,
    pub enableLdm: c_int,
    pub windowLog: c_uint,
    pub chainLog: c_uint,
    pub strategy: c_int,
    pub overlapLog: c_int,
    pub rsyncable: c_int,
    pub roundBuffCapacity: usize,
    pub allJobsCompleted: c_uint,
}

pub type ZSTDMT_initResizeFn = unsafe extern "C" fn(*mut c_void, c_uint) -> usize;
pub type ZSTDMT_initWaitForAllJobsFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_initReleaseAllJobResourcesFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_initApplyParametersFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_initDictionaryFn = unsafe extern "C" fn(*mut c_void) -> usize;
pub type ZSTDMT_initSetSizeFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_initSetRsyncFn = unsafe extern "C" fn(*mut c_void, u64, u64);
pub type ZSTDMT_initSetBufferSizeFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_initResizeRoundBufferFn = unsafe extern "C" fn(*mut c_void, usize) -> usize;
pub type ZSTDMT_initResetStreamFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_initSerialResetFn = unsafe extern "C" fn(*mut c_void, usize) -> usize;

/// Scalar dictionary classification for the MT stream initializer. C keeps
/// dictionary handles, parameter layout, prefix storage, and CDict mutation
/// behind callbacks; Rust owns the copy/borrow/raw-prefix branch selection.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_RustDictionaryProjection {
    pub hasDictionary: c_uint,
    pub rawContent: c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_RustDictionaryProjection, hasDictionary) == 0);
    assert!(offset_of!(ZSTDMT_RustDictionaryProjection, rawContent) == size_of::<c_uint>());
    assert!(size_of::<ZSTDMT_RustDictionaryProjection>() == 2 * size_of::<c_uint>());
};

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum PrepareDictionaryAction {
    CreateCopied,
    AttachBorrowed,
}

#[inline]
fn classify_prepare_dictionary(
    projection: ZSTDMT_RustDictionaryProjection,
) -> PrepareDictionaryAction {
    if projection.hasDictionary != 0 {
        PrepareDictionaryAction::CreateCopied
    } else {
        PrepareDictionaryAction::AttachBorrowed
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum UpdateDictionaryAction {
    AttachBorrowed,
    SetRawPrefix,
    CreateReferenced,
}

#[inline]
fn classify_update_dictionary(
    projection: ZSTDMT_RustDictionaryProjection,
) -> UpdateDictionaryAction {
    if projection.hasDictionary == 0 {
        UpdateDictionaryAction::AttachBorrowed
    } else if projection.rawContent != 0 {
        UpdateDictionaryAction::SetRawPrefix
    } else {
        UpdateDictionaryAction::CreateReferenced
    }
}

#[inline]
fn prepare_c_stream_dictionary_with<R, C, A>(
    projection: ZSTDMT_RustDictionaryProjection,
    mut release_local: R,
    mut create_copied: C,
    mut attach_borrowed: A,
) -> usize
where
    R: FnMut(),
    C: FnMut() -> usize,
    A: FnMut(),
{
    release_local();
    match classify_prepare_dictionary(projection) {
        PrepareDictionaryAction::CreateCopied => create_copied(),
        PrepareDictionaryAction::AttachBorrowed => {
            attach_borrowed();
            0
        }
    }
}

#[inline]
fn update_c_stream_dictionary_with<C, A, P, R>(
    projection: ZSTDMT_RustDictionaryProjection,
    mut clear_local: C,
    mut attach_borrowed: A,
    mut set_raw_prefix: P,
    mut create_referenced: R,
) -> usize
where
    C: FnMut(),
    A: FnMut(),
    P: FnMut(),
    R: FnMut() -> usize,
{
    clear_local();
    match classify_update_dictionary(projection) {
        UpdateDictionaryAction::AttachBorrowed => {
            attach_borrowed();
            0
        }
        UpdateDictionaryAction::SetRawPrefix => {
            set_raw_prefix();
            0
        }
        UpdateDictionaryAction::CreateReferenced => create_referenced(),
    }
}

/// Apply the MT stream dictionary transition policy while C retains all
/// private CDict allocation and context publication operations.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_prepareCStreamDictionary(
    projection: *const ZSTDMT_RustDictionaryProjection,
    opaque: *mut c_void,
    release_local: Option<ZSTDMT_initResetStreamFn>,
    create_copied: Option<ZSTDMT_initDictionaryFn>,
    attach_borrowed: Option<ZSTDMT_initResetStreamFn>,
) -> usize {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ERROR(ZstdErrorCode::Generic);
    };
    let (Some(release_local), Some(create_copied), Some(attach_borrowed)) =
        (release_local, create_copied, attach_borrowed)
    else {
        return ERROR(ZstdErrorCode::Generic);
    };
    if opaque.is_null() {
        return ERROR(ZstdErrorCode::Generic);
    }

    prepare_c_stream_dictionary_with(
        projection,
        || unsafe { release_local(opaque) },
        || unsafe { create_copied(opaque) },
        || unsafe { attach_borrowed(opaque) },
    )
}

/// Apply the MT stream dictionary update policy while C retains private raw
/// prefix and CDict state. The clear step always precedes the selected branch.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_updateCStreamDictionary(
    projection: *const ZSTDMT_RustDictionaryProjection,
    opaque: *mut c_void,
    clear_local: Option<ZSTDMT_initResetStreamFn>,
    attach_borrowed: Option<ZSTDMT_initResetStreamFn>,
    set_raw_prefix: Option<ZSTDMT_initResetStreamFn>,
    create_referenced: Option<ZSTDMT_initDictionaryFn>,
) -> usize {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ERROR(ZstdErrorCode::Generic);
    };
    let (Some(clear_local), Some(attach_borrowed), Some(set_raw_prefix), Some(create_referenced)) = (
        clear_local,
        attach_borrowed,
        set_raw_prefix,
        create_referenced,
    ) else {
        return ERROR(ZstdErrorCode::Generic);
    };
    if opaque.is_null() {
        return ERROR(ZstdErrorCode::Generic);
    }

    update_c_stream_dictionary_with(
        projection,
        || unsafe { clear_local(opaque) },
        || unsafe { attach_borrowed(opaque) },
        || unsafe { set_raw_prefix(opaque) },
        || unsafe { create_referenced(opaque) },
    )
}

type ZSTDMT_resetStreamCallbackFn = unsafe extern "C" fn(*mut c_void);
/// Scalar inputs for the MT serial-state reset. The private C parameter,
/// serial-state, LDM-state, and sequence-pool layouts stay behind callbacks;
/// Rust owns only their scalar policy and ordering.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_RustSerialResetProjection {
    pub enableLdm: c_int,
    pub checksumEnabled: c_int,
    pub jobSize: usize,
    pub windowLog: c_uint,
    pub strategy: c_int,
    pub hashLog: c_uint,
    pub bucketSizeLog: c_uint,
    pub minMatchLength: c_uint,
    pub hashRateLog: c_uint,
    pub previousHashLog: c_uint,
    pub previousBucketSizeLog: c_uint,
    pub ldmEntrySize: usize,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_RustSerialResetProjection, enableLdm) == 0);
    assert!(offset_of!(ZSTDMT_RustSerialResetProjection, checksumEnabled) == size_of::<c_int>());
    assert!(offset_of!(ZSTDMT_RustSerialResetProjection, jobSize) == 8);
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, windowLog)
            == if size_of::<usize>() == 8 { 16 } else { 12 }
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, strategy)
            == if size_of::<usize>() == 8 { 20 } else { 16 }
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, hashLog)
            == if size_of::<usize>() == 8 { 24 } else { 20 }
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, bucketSizeLog)
            == if size_of::<usize>() == 8 { 28 } else { 24 }
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, minMatchLength)
            == if size_of::<usize>() == 8 { 32 } else { 28 }
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, hashRateLog)
            == if size_of::<usize>() == 8 { 36 } else { 32 }
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, previousHashLog)
            == if size_of::<usize>() == 8 { 40 } else { 36 }
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, previousBucketSizeLog)
            == if size_of::<usize>() == 8 { 44 } else { 40 }
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialResetProjection, ldmEntrySize)
            == if size_of::<usize>() == 8 { 48 } else { 44 }
    );
    assert!(
        size_of::<ZSTDMT_RustSerialResetProjection>()
            == if size_of::<usize>() == 8 { 56 } else { 48 }
    );
};

pub type ZSTDMT_serialResetSetLdmParamsFn =
    unsafe extern "C" fn(*mut c_void, c_int, c_uint, c_uint, c_uint, c_uint, c_uint);
pub type ZSTDMT_serialResetVoidFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialResetSetNbSeqFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_serialResetResizeFn =
    unsafe extern "C" fn(*mut c_void, usize, usize, c_uint, c_uint) -> c_int;
pub type ZSTDMT_serialResetZeroTablesFn = unsafe extern "C" fn(*mut c_void, usize, usize);
pub type ZSTDMT_serialResetLoadRawDictionaryFn =
    unsafe extern "C" fn(*mut c_void, *const c_void, usize) -> usize;
pub type ZSTDMT_serialResetSetLoadedDictEndFn = unsafe extern "C" fn(*mut c_void, c_uint);
pub type ZSTDMT_serialResetPublishFn = unsafe extern "C" fn(*mut c_void, usize);
pub type ZSTDMT_serialStateVoidFn = unsafe extern "C" fn(*mut c_void);
pub type ZSTDMT_serialStateInitFn = unsafe extern "C" fn(*mut c_void) -> c_int;

/// Dictionary inputs for the MT serial LDM preload policy. The private C
/// window/hash-table state stays behind callbacks; Rust owns eligibility,
/// reset, and the `forceWindow` publication rule.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_RustSerialDictionaryProjection {
    pub dict: *const c_void,
    pub dictSize: usize,
    pub dictContentType: c_int,
    pub forceWindow: c_int,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_RustSerialDictionaryProjection, dict) == 0);
    assert!(
        offset_of!(ZSTDMT_RustSerialDictionaryProjection, dictSize) == size_of::<*const c_void>()
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialDictionaryProjection, dictContentType)
            == size_of::<*const c_void>() + size_of::<usize>()
    );
    assert!(
        offset_of!(ZSTDMT_RustSerialDictionaryProjection, forceWindow)
            == size_of::<*const c_void>() + size_of::<usize>() + size_of::<c_int>()
    );
    assert!(
        size_of::<ZSTDMT_RustSerialDictionaryProjection>()
            == size_of::<*const c_void>() + size_of::<usize>() + 2 * size_of::<c_int>()
    );
};

/// Rust's LDM module already owns the scalar adjustment and maximum-sequence
/// leaves. This mirror is Rust-internal: it contains only the six scalar LDM
/// fields needed to call those leaves and is never exposed in the C ABI.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct ZSTDMT_serialResetLdmParameters {
    enableLdm: c_int,
    hashLog: c_uint,
    bucketSizeLog: c_uint,
    minMatchLength: c_uint,
    hashRateLog: c_uint,
    windowLog: c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_serialResetLdmParameters, enableLdm) == 0);
    assert!(offset_of!(ZSTDMT_serialResetLdmParameters, hashLog) == size_of::<c_int>());
    assert!(
        offset_of!(ZSTDMT_serialResetLdmParameters, bucketSizeLog)
            == size_of::<c_int>() + size_of::<c_uint>()
    );
    assert!(
        offset_of!(ZSTDMT_serialResetLdmParameters, minMatchLength)
            == size_of::<c_int>() + 2 * size_of::<c_uint>()
    );
    assert!(
        offset_of!(ZSTDMT_serialResetLdmParameters, hashRateLog)
            == size_of::<c_int>() + 3 * size_of::<c_uint>()
    );
    assert!(
        offset_of!(ZSTDMT_serialResetLdmParameters, windowLog)
            == size_of::<c_int>() + 4 * size_of::<c_uint>()
    );
    assert!(size_of::<ZSTDMT_serialResetLdmParameters>() == 6 * size_of::<c_uint>());
};

#[inline]
fn serial_state_load_dictionary_with<L, S>(
    projection: ZSTDMT_RustSerialDictionaryProjection,
    mut load_raw_dictionary: L,
    mut set_loaded_dict_end: S,
) where
    L: FnMut(*const c_void, usize) -> usize,
    S: FnMut(c_uint),
{
    /* Match the C reset: loadedDictEnd is cleared even when no raw
     * dictionary is eligible for loading. */
    set_loaded_dict_end(0);
    if projection.dictSize == 0 || projection.dictContentType != ZSTD_DCT_RAW_CONTENT {
        return;
    }

    let loaded_dict_end = load_raw_dictionary(projection.dict, projection.dictSize);
    if projection.forceWindow == 0 {
        set_loaded_dict_end(loaded_dict_end as c_uint);
    }
}

/// Apply the MT serial LDM dictionary preload policy. C retains the private
/// window/hash-table operations and publishes their computed offset through
/// the explicit callbacks.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateLoadDictionary(
    projection: *const ZSTDMT_RustSerialDictionaryProjection,
    opaque: *mut c_void,
    load_raw_dictionary: Option<ZSTDMT_serialResetLoadRawDictionaryFn>,
    set_loaded_dict_end: Option<ZSTDMT_serialResetSetLoadedDictEndFn>,
) {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return;
    };
    let (Some(load_raw_dictionary), Some(set_loaded_dict_end)) =
        (load_raw_dictionary, set_loaded_dict_end)
    else {
        return;
    };
    if opaque.is_null() {
        return;
    }

    unsafe {
        serial_state_load_dictionary_with(
            projection,
            |dict, dict_size| load_raw_dictionary(opaque, dict, dict_size),
            |loaded_dict_end| set_loaded_dict_end(opaque, loaded_dict_end),
        );
    }
}

unsafe extern "C" {
    fn ZSTD_ldm_adjustParameters(
        params: *mut ZSTDMT_serialResetLdmParameters,
        cParams: *const ZSTD_compressionParameters,
    );
    fn ZSTD_ldm_getMaxNbSeq(params: ZSTDMT_serialResetLdmParameters, maxChunkSize: usize) -> usize;
}

/// Callback projection for the MT stream reset. C retains the private buffer,
/// job, flag, and progress fields; Rust owns the order in which each group is
/// reset.
#[repr(C)]
pub struct ZSTDMT_RustResetStreamState {
    callback_context: *mut c_void,
    reset_round_buffer: Option<ZSTDMT_resetStreamCallbackFn>,
    reset_input_buffer: Option<ZSTDMT_resetStreamCallbackFn>,
    reset_job_ids: Option<ZSTDMT_resetStreamCallbackFn>,
    reset_frame_flags: Option<ZSTDMT_resetStreamCallbackFn>,
    reset_progress: Option<ZSTDMT_resetStreamCallbackFn>,
}

const _: () = {
    assert!(size_of::<ZSTDMT_resetStreamCallbackFn>() == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustResetStreamState, callback_context) == 0);
    assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_round_buffer) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_input_buffer) == 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_job_ids) == 3 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_frame_flags) == 4 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustResetStreamState, reset_progress) == 5 * size_of::<usize>());
    assert!(size_of::<ZSTDMT_RustResetStreamState>() == 6 * size_of::<usize>());
};

/// Reset the MT stream state in the same order as the original C function.
/// Each callback keeps the corresponding private C layout and sentinel values
/// out of the Rust ABI.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_resetStream(state: *const ZSTDMT_RustResetStreamState) {
    let Some(state) = (unsafe { state.as_ref() }) else {
        return;
    };
    if state.callback_context.is_null() {
        return;
    }
    let (
        Some(reset_round_buffer),
        Some(reset_input_buffer),
        Some(reset_job_ids),
        Some(reset_frame_flags),
        Some(reset_progress),
    ) = (
        state.reset_round_buffer,
        state.reset_input_buffer,
        state.reset_job_ids,
        state.reset_frame_flags,
        state.reset_progress,
    )
    else {
        return;
    };

    unsafe {
        reset_round_buffer(state.callback_context);
        reset_input_buffer(state.callback_context);
        reset_job_ids(state.callback_context);
        reset_frame_flags(state.callback_context);
        reset_progress(state.callback_context);
    }
}

#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_flushPublicationResult {
    pub status: c_int,
    pub toFlush: usize,
    pub outputPos: usize,
    pub dstFlushed: usize,
}

/// Scalar snapshot of the oldest active C-owned job used by the flush state
/// machine.  C keeps the descriptor, mutex, buffer pool, and checksum state;
/// Rust decides when this snapshot can be published and completed.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_flushJobProjection {
    pub consumed: usize,
    pub cSize: usize,
    pub srcSize: usize,
    pub dstStart: *const c_void,
    pub dstCapacity: usize,
    pub dstFlushed: usize,
    pub frameChecksumNeeded: c_uint,
}

/// Narrow projection for serializing the C-owned frame checksum.  C retains
/// the serial XXH64 state and computes the checksum; Rust owns only the
/// deterministic little-endian write and destination policy.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_frameChecksumProjection {
    pub dst: *mut c_void,
    pub dstCapacity: usize,
    pub cSize: usize,
    pub checksum: c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_frameChecksumProjection, dst) == 0);
    assert!(offset_of!(ZSTDMT_frameChecksumProjection, dstCapacity) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_frameChecksumProjection, cSize) == 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_frameChecksumProjection, checksum) == 3 * size_of::<usize>());
    assert!(size_of::<c_uint>() == 4);
    assert!(size_of::<ZSTDMT_frameChecksumProjection>() == 4 * size_of::<usize>());
};

#[inline]
fn write_frame_checksum(projection: &ZSTDMT_frameChecksumProjection) -> usize {
    if projection.dst.is_null()
        || projection.dstCapacity < 4
        || projection.cSize > projection.dstCapacity - 4
    {
        return ERROR(ZstdErrorCode::DstSizeTooSmall);
    }

    unsafe {
        MEM_writeLE32(
            projection.dst.cast::<u8>().add(projection.cSize).cast(),
            projection.checksum,
        );
    }
    projection.cSize + 4
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_writeFrameChecksum(
    projection: *const ZSTDMT_frameChecksumProjection,
) -> usize {
    let Some(projection) = (unsafe { projection.as_ref() }) else {
        return ERROR(ZstdErrorCode::Generic);
    };
    write_frame_checksum(projection)
}

/// Scalar view of the C-owned MT state needed after the current job is
/// published.  The C adapter applies descriptor and context mutations made by
/// Rust's decisions through callbacks.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_flushContextProjection {
    pub doneJobID: c_uint,
    pub nextJobID: c_uint,
    pub jobIDMask: c_uint,
    pub jobReady: c_uint,
    pub frameEnded: c_uint,
    pub inBuffFilled: usize,
}

#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_flushProducedResult {
    pub result: usize,
    pub outputPos: usize,
    pub updateAllJobsCompleted: c_uint,
    pub allJobsCompleted: c_uint,
}

pub type ZSTDMT_flushProjectJobFn = unsafe extern "C" fn(
    opaque: *mut c_void,
    job_id: c_uint,
    block_to_flush: c_uint,
    projection: *mut ZSTDMT_flushJobProjection,
);
pub type ZSTDMT_flushChecksumFn = unsafe extern "C" fn(
    opaque: *mut c_void,
    job_id: c_uint,
    projection: *mut ZSTDMT_flushJobProjection,
);
pub type ZSTDMT_flushUpdateJobFn =
    unsafe extern "C" fn(opaque: *mut c_void, job_id: c_uint, dst_flushed: usize);
pub type ZSTDMT_flushCompleteJobFn =
    unsafe extern "C" fn(opaque: *mut c_void, job_id: c_uint, src_size: usize, c_size: usize);
pub type ZSTDMT_flushWaitForAllJobsFn = unsafe extern "C" fn(opaque: *mut c_void);
pub type ZSTDMT_flushReleaseAllJobResourcesFn = unsafe extern "C" fn(opaque: *mut c_void);

/// Scalar MT state used by the outer streaming scheduler.  The C context,
/// input-range ownership, and synchronization objects remain private to C;
/// Rust only reads the fields needed to choose the next operation.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressStreamContextProjection {
    pub frameEnded: c_uint,
    pub jobReady: c_uint,
    pub inBuffStart: *mut c_void,
    pub inBuffCapacity: usize,
    pub inBuffFilled: usize,
    pub targetSectionSize: usize,
    pub rsyncable: c_int,
    pub rsyncPrimePower: u64,
    pub rsyncHitMask: u64,
}

/// Scalar view of the reusable C-owned input range returned by the existing
/// MT input-range seam.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamInputRangeProjection {
    pub bufferStart: *mut c_void,
    pub bufferCapacity: usize,
    pub bufferFilled: usize,
}

/// Explicit projections of the public stream buffers.  These deliberately do
/// not use the private C context layout or rely on a Rust view of `ZSTD_inBuffer`
/// and `ZSTD_outBuffer`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamInputProjection {
    pub src: *const c_void,
    pub size: usize,
    pub pos: usize,
}

#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamOutputProjection {
    pub dst: *mut c_void,
    pub size: usize,
    pub pos: usize,
}

#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_syncPointProjection {
    pub toLoad: usize,
    pub flush: c_int,
}

/// Result returned by the C-owned pending-output seam after it has updated a
/// temporary output projection.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamFlushResult {
    pub result: usize,
    pub outputPos: usize,
}

/// Result of the C-owned job callback.  C reports only whether its private
/// input range remains active; Rust owns the policy that publishes its local
/// `inBuffFilled` value after the callback.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_streamCreateJobResult {
    pub result: usize,
    pub inBuffActive: c_uint,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_streamCreateJobResult, result) == 0);
    assert!(offset_of!(ZSTDMT_streamCreateJobResult, inBuffActive) == size_of::<usize>());
    assert!(size_of::<ZSTDMT_streamCreateJobResult>() == 2 * size_of::<usize>());
};

/// Result of one outer MT stream scheduling pass.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_compressStreamResult {
    pub result: usize,
    pub inputPos: usize,
    pub outputPos: usize,
    pub inBuffFilled: usize,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_compressStreamResult, result) == 0);
    assert!(offset_of!(ZSTDMT_compressStreamResult, inputPos) == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressStreamResult, outputPos) == 2 * size_of::<usize>());
    assert!(offset_of!(ZSTDMT_compressStreamResult, inBuffFilled) == 3 * size_of::<usize>());
    assert!(size_of::<ZSTDMT_compressStreamResult>() == 4 * size_of::<usize>());
};

pub type ZSTDMT_streamTryGetInputRangeFn = unsafe extern "C" fn(
    opaque: *mut c_void,
    projection: *mut ZSTDMT_streamInputRangeProjection,
) -> c_int;
pub type ZSTDMT_streamCreateJobFn = unsafe extern "C" fn(
    opaque: *mut c_void,
    src_size: usize,
    end: c_uint,
) -> ZSTDMT_streamCreateJobResult;
pub type ZSTDMT_streamFlushProducedFn = unsafe extern "C" fn(
    opaque: *mut c_void,
    output_dst: *mut c_void,
    output_size: usize,
    output_pos: usize,
    block_to_flush: c_uint,
    end: c_uint,
) -> ZSTDMT_streamFlushResult;

const ZSTD_E_END: c_uint = 2;
const ZSTD_E_FLUSH: c_uint = 1;
const ZSTD_E_CONTINUE: c_uint = 0;

/// Scalar view of the C-owned job state needed to finish a streaming frame
/// with an empty last block.  The job descriptor, buffer pool, and mutex stay
/// private to C; Rust receives only this projection and a buffer callback.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_emptyBlockJobProjection {
    pub lastJob: c_uint,
    pub srcSize: usize,
    pub firstJob: c_uint,
    pub dstStart: *mut c_void,
    pub dstCapacity: usize,
    pub consumed: usize,
}

#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZSTDMT_emptyBlockResult {
    pub buffer: ZstdMtBuffer,
    pub cSize: usize,
    pub clearSource: c_uint,
}

pub type ZSTDMT_bufferGetFn = unsafe extern "C" fn(*mut c_void) -> ZstdMtBuffer;

/// Scalar snapshot of one C-owned job descriptor.
///
/// The C adapter fills the complete projection while holding the descriptor
/// mutex. Rust owns only the ring-scan decisions below; the descriptor layout,
/// synchronization objects, and all pointer ownership stay in C.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_jobProjection {
    pub consumed: usize,
    pub cSize: usize,
    pub srcStart: *const c_void,
    pub srcSize: usize,
    pub prefixStart: *const c_void,
    pub prefixSize: usize,
    pub dstFlushed: usize,
}

#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_inputRange {
    pub start: *const c_void,
    pub size: usize,
}

/// Scalar input-buffer state used by the MT range-selection policy.  C keeps
/// the input and round-buffer structs private; Rust decides only whether the
/// next target section can be reused and where that section starts.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_tryGetInputRangeProjection {
    pub roundBufferStart: *const c_void,
    pub roundBufferCapacity: usize,
    pub roundBufferPos: usize,
    pub prefixStart: *const c_void,
    pub prefixSize: usize,
    pub targetSectionSize: usize,
    pub inUseStart: *const c_void,
    pub inUseSize: usize,
}

/// Result of the MT input-range selection. Rust also owns the ordering of the
/// synchronization waits and publication callbacks that commit this result;
/// C retains the private buffer mutations and synchronization state.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_tryGetInputRangeResult {
    pub ready: c_uint,
    pub movePrefix: c_uint,
    pub bufferStart: *mut c_void,
    pub bufferCapacity: usize,
    pub roundBufferPos: usize,
}

const _: () = {
    assert!(offset_of!(ZSTDMT_tryGetInputRangeResult, ready) == 0);
    assert!(offset_of!(ZSTDMT_tryGetInputRangeResult, movePrefix) == size_of::<c_uint>());
    assert!(offset_of!(ZSTDMT_tryGetInputRangeResult, bufferStart) == 2 * size_of::<c_uint>());
    assert!(
        offset_of!(ZSTDMT_tryGetInputRangeResult, bufferCapacity)
            == 2 * size_of::<c_uint>() + size_of::<*mut c_void>()
    );
    assert!(
        offset_of!(ZSTDMT_tryGetInputRangeResult, roundBufferPos)
            == 2 * size_of::<c_uint>() + size_of::<*mut c_void>() + size_of::<usize>()
    );
    assert!(
        size_of::<ZSTDMT_tryGetInputRangeResult>()
            == 2 * size_of::<c_uint>() + size_of::<*mut c_void>() + 2 * size_of::<usize>()
    );
};

pub type ZSTDMT_inputRangeWaitFn =
    unsafe extern "C" fn(opaque: *mut c_void, buffer_start: *mut c_void, buffer_capacity: usize);
pub type ZSTDMT_inputRangeMovePrefixFn = unsafe extern "C" fn(
    opaque: *mut c_void,
    destination: *mut c_void,
    source: *const c_void,
    size: usize,
    round_buffer_pos: usize,
);
pub type ZSTDMT_inputRangePublishBufferFn =
    unsafe extern "C" fn(opaque: *mut c_void, buffer_start: *mut c_void, buffer_capacity: usize);

#[inline]
fn commit_input_range_with<W, M, P>(
    result: ZSTDMT_tryGetInputRangeResult,
    round_buffer_start: *mut c_void,
    prefix_start: *const c_void,
    prefix_size: usize,
    opaque: *mut c_void,
    mut wait_for_ldm: W,
    mut move_prefix: M,
    mut publish_buffer: P,
) -> c_int
where
    W: FnMut(*mut c_void, *mut c_void, usize),
    M: FnMut(*mut c_void, *mut c_void, *const c_void, usize, usize),
    P: FnMut(*mut c_void, *mut c_void, usize),
{
    if result.ready == 0 {
        return 0;
    }

    if result.movePrefix != 0 {
        wait_for_ldm(opaque, round_buffer_start, prefix_size);
        move_prefix(
            opaque,
            round_buffer_start,
            prefix_start,
            prefix_size,
            result.roundBufferPos,
        );
    }

    wait_for_ldm(opaque, result.bufferStart, result.bufferCapacity);
    publish_buffer(opaque, result.bufferStart, result.bufferCapacity);
    1
}

/// Commit the selected MT input range in the original order. Rust owns the
/// prefix-wrap branch and the wait/copy/wait/publish sequencing; C callbacks
/// retain the private round-buffer, input-buffer, and LDM synchronization
/// operations.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_commitInputRange(
    result: *const ZSTDMT_tryGetInputRangeResult,
    roundBufferStart: *mut c_void,
    prefixStart: *const c_void,
    prefixSize: usize,
    opaque: *mut c_void,
    waitForLdm: Option<ZSTDMT_inputRangeWaitFn>,
    movePrefix: Option<ZSTDMT_inputRangeMovePrefixFn>,
    publishBuffer: Option<ZSTDMT_inputRangePublishBufferFn>,
) -> c_int {
    let Some(result) = (unsafe { result.as_ref() }).copied() else {
        return 0;
    };
    let (Some(wait_for_ldm), Some(move_prefix), Some(publish_buffer)) =
        (waitForLdm, movePrefix, publishBuffer)
    else {
        return 0;
    };
    if opaque.is_null() {
        return 0;
    }

    commit_input_range_with(
        result,
        roundBufferStart,
        prefixStart,
        prefixSize,
        opaque,
        |opaque, buffer_start, buffer_capacity| unsafe {
            wait_for_ldm(opaque, buffer_start, buffer_capacity);
        },
        |opaque, destination, source, size, round_buffer_pos| unsafe {
            move_prefix(opaque, destination, source, size, round_buffer_pos);
        },
        |opaque, buffer_start, buffer_capacity| unsafe {
            publish_buffer(opaque, buffer_start, buffer_capacity);
        },
    )
}

pub type ZSTDMT_jobProjectionFn = unsafe extern "C" fn(
    opaque: *mut c_void,
    job_id: c_uint,
    projection: *mut ZSTDMT_jobProjection,
);

const FLUSH_PUBLICATION_OK: c_int = 0;
const FLUSH_PUBLICATION_INVALID_BOUNDS: c_int = 1;

#[inline]
fn empty_block_allocation_error() -> ZSTDMT_emptyBlockResult {
    ZSTDMT_emptyBlockResult {
        buffer: ZstdMtBuffer::default(),
        cSize: ERROR(ZstdErrorCode::MemoryAllocation),
        clearSource: 0,
    }
}

#[inline]
unsafe fn write_last_empty_block_with<F>(
    projection: ZSTDMT_emptyBlockJobProjection,
    mut get_buffer: F,
) -> ZSTDMT_emptyBlockResult
where
    F: FnMut() -> ZstdMtBuffer,
{
    debug_assert_eq!(projection.lastJob, 1);
    debug_assert_eq!(projection.srcSize, 0);
    debug_assert_eq!(projection.firstJob, 0);
    debug_assert!(projection.dstStart.is_null());
    debug_assert_eq!(projection.dstCapacity, 0);
    debug_assert_eq!(projection.consumed, 0);

    let buffer = get_buffer();
    if buffer.start.is_null() {
        return empty_block_allocation_error();
    }

    let c_size = unsafe {
        crate::zstd_compress_frame::ZSTD_writeLastEmptyBlock(buffer.start, buffer.capacity)
    };
    ZSTDMT_emptyBlockResult {
        buffer,
        cSize: c_size,
        /* The C implementation clears src immediately after acquiring the
         * buffer, before serializing the block. */
        clearSource: 1,
    }
}

/// Acquire and serialize the terminal empty block for a C-owned MT job.
///
/// C keeps the private descriptor and applies the returned state after this
/// callback-driven operation.  In particular, an allocation failure leaves
/// the source range untouched just as the original C helper did.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_writeLastEmptyBlock(
    projection: *const ZSTDMT_emptyBlockJobProjection,
    opaque: *mut c_void,
    getBuffer: Option<ZSTDMT_bufferGetFn>,
) -> ZSTDMT_emptyBlockResult {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return empty_block_allocation_error();
    };
    let Some(get_buffer) = getBuffer else {
        return empty_block_allocation_error();
    };
    unsafe { write_last_empty_block_with(projection, || get_buffer(opaque)) }
}

const CREATE_JOB_TABLE_FULL: c_uint = 0;
const CREATE_JOB_POST: c_uint = 1;
const CREATE_JOB_EMPTY: c_uint = 2;

/// Scalar MT state supplied by C for one compression-job creation attempt.
///
/// The job descriptor, input-buffer ownership, and synchronization objects
/// remain private to C. Rust owns the ring-capacity and scheduling policy over
/// this scalar view.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_createJobProjection {
    pub doneJobID: c_uint,
    pub nextJobID: c_uint,
    pub jobIDMask: c_uint,
    pub jobReady: c_uint,
    pub srcStart: *const c_void,
    pub srcSize: usize,
    pub inBuffFilled: usize,
    pub prefixStart: *const c_void,
    pub prefixSize: usize,
    pub targetPrefixSize: usize,
    pub endFrame: c_uint,
    pub checksumFlag: c_uint,
}

/// The scalar fields needed by C to initialize its private job descriptor and
/// the surrounding input state after Rust chooses to prepare a new job.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_jobInitialization {
    pub srcStart: *const c_void,
    pub srcSize: usize,
    pub prefixStart: *const c_void,
    pub prefixSize: usize,
    pub nextPrefixStart: *const c_void,
    pub nextPrefixSize: usize,
    pub roundBuffPosDelta: usize,
    pub jobNumber: c_uint,
    pub firstJob: c_uint,
    pub lastJob: c_uint,
    pub frameChecksumNeeded: c_uint,
    pub clearChecksumFlag: c_uint,
}

#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_createJobResult {
    pub returnCode: usize,
    pub action: c_uint,
    pub jobID: c_uint,
    pub jobNumber: c_uint,
    pub nextJobID: c_uint,
    pub jobReady: c_uint,
}

pub type ZSTDMT_prepareJobFn = unsafe extern "C" fn(
    opaque: *mut c_void,
    job_id: c_uint,
    initialization: *const ZSTDMT_jobInitialization,
);
pub type ZSTDMT_prepareFrameStateFn = unsafe extern "C" fn(opaque: *mut c_void);
pub type ZSTDMT_writeEmptyJobFn = unsafe extern "C" fn(opaque: *mut c_void, job_id: c_uint);
pub type ZSTDMT_tryAddJobFn = unsafe extern "C" fn(opaque: *mut c_void, job_id: c_uint) -> c_int;

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum CompressionJobFrameStateAction {
    None,
    SetFrameEnded,
    SetFrameEndedAndClearChecksum,
}

#[inline]
fn classify_compression_job_frame_state(
    last_job: c_uint,
    clear_checksum_flag: c_uint,
) -> CompressionJobFrameStateAction {
    if last_job == 0 {
        CompressionJobFrameStateAction::None
    } else if clear_checksum_flag == 0 {
        CompressionJobFrameStateAction::SetFrameEnded
    } else {
        CompressionJobFrameStateAction::SetFrameEndedAndClearChecksum
    }
}

/// Order the C-owned leaves that publish a newly prepared MT job.
///
/// The descriptor must become complete before the reusable input range is
/// cleared, and terminal frame state is published last. `lastJob` and
/// `clearChecksumFlag` are classified in Rust; C callbacks retain the private
/// context mutations.
#[inline]
fn prepare_compression_job_with<P, I, S, C>(
    job_id: c_uint,
    initialization: &ZSTDMT_jobInitialization,
    mut initialize_job: P,
    mut reset_input_state: I,
    mut set_frame_ended: S,
    mut clear_checksum_flag: C,
) where
    P: FnMut(c_uint, &ZSTDMT_jobInitialization),
    I: FnMut(c_uint, &ZSTDMT_jobInitialization),
    S: FnMut(),
    C: FnMut(),
{
    let frame_state_action = classify_compression_job_frame_state(
        initialization.lastJob,
        initialization.clearChecksumFlag,
    );

    initialize_job(job_id, initialization);
    reset_input_state(job_id, initialization);
    match frame_state_action {
        CompressionJobFrameStateAction::None => {}
        CompressionJobFrameStateAction::SetFrameEnded => set_frame_ended(),
        CompressionJobFrameStateAction::SetFrameEndedAndClearChecksum => {
            set_frame_ended();
            clear_checksum_flag();
        }
    }
}

/// Apply the MT job-preparation ordering while C retains the private job
/// descriptor and stream-state fields behind callbacks.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_prepareCompressionJob(
    opaque: *mut c_void,
    job_id: c_uint,
    initialization: *const ZSTDMT_jobInitialization,
    initialize_job: Option<ZSTDMT_prepareJobFn>,
    reset_input_state: Option<ZSTDMT_prepareJobFn>,
    set_frame_ended: Option<ZSTDMT_prepareFrameStateFn>,
    clear_checksum_flag: Option<ZSTDMT_prepareFrameStateFn>,
) {
    let Some(initialization) = (unsafe { initialization.as_ref() }) else {
        return;
    };
    let (
        Some(initialize_job),
        Some(reset_input_state),
        Some(set_frame_ended),
        Some(clear_checksum_flag),
    ) = (
        initialize_job,
        reset_input_state,
        set_frame_ended,
        clear_checksum_flag,
    )
    else {
        return;
    };
    if opaque.is_null() {
        return;
    }

    prepare_compression_job_with(
        job_id,
        initialization,
        |job_id, initialization| unsafe {
            initialize_job(opaque, job_id, initialization);
        },
        |job_id, initialization| unsafe {
            reset_input_state(opaque, job_id, initialization);
        },
        || unsafe {
            set_frame_ended(opaque);
        },
        || unsafe {
            clear_checksum_flag(opaque);
        },
    );
}

#[inline]
fn create_job_table_full(projection: ZSTDMT_createJobProjection) -> ZSTDMT_createJobResult {
    ZSTDMT_createJobResult {
        action: CREATE_JOB_TABLE_FULL,
        jobID: projection.nextJobID & projection.jobIDMask,
        jobNumber: projection.nextJobID,
        nextJobID: projection.nextJobID,
        jobReady: projection.jobReady,
        ..ZSTDMT_createJobResult::default()
    }
}

#[inline]
fn create_job_initialization(
    projection: ZSTDMT_createJobProjection,
) -> (c_uint, ZSTDMT_jobInitialization) {
    let job_number = projection.nextJobID;
    let first_job = (job_number == 0) as c_uint;
    let last_job = (projection.endFrame != 0) as c_uint;
    let frame_checksum_needed =
        (projection.checksumFlag != 0 && last_job != 0 && job_number > 0) as c_uint;

    let (next_prefix_start, next_prefix_size) = if last_job != 0 {
        (ptr::null(), 0)
    } else {
        let next_prefix_size = projection.srcSize.min(projection.targetPrefixSize);
        let next_prefix_start = projection
            .srcStart
            .cast::<u8>()
            .wrapping_add(projection.srcSize - next_prefix_size)
            .cast();
        (next_prefix_start, next_prefix_size)
    };

    (
        projection.nextJobID & projection.jobIDMask,
        ZSTDMT_jobInitialization {
            srcStart: projection.srcStart,
            srcSize: projection.srcSize,
            prefixStart: projection.prefixStart,
            prefixSize: projection.prefixSize,
            nextPrefixStart: next_prefix_start,
            nextPrefixSize: next_prefix_size,
            roundBuffPosDelta: projection.srcSize,
            jobNumber: job_number,
            firstJob: first_job,
            lastJob: last_job,
            frameChecksumNeeded: frame_checksum_needed,
            clearChecksumFlag: (last_job != 0 && first_job != 0) as c_uint,
        },
    )
}

#[inline]
fn create_compression_job_with<P, E, T>(
    projection: ZSTDMT_createJobProjection,
    mut prepare_job: P,
    mut write_empty_job: E,
    mut try_add_job: T,
) -> ZSTDMT_createJobResult
where
    P: FnMut(c_uint, &ZSTDMT_jobInitialization),
    E: FnMut(c_uint),
    T: FnMut(c_uint) -> bool,
{
    /* Match the original unsigned C comparison, including wraparound. */
    if projection.nextJobID > projection.doneJobID.wrapping_add(projection.jobIDMask) {
        return create_job_table_full(projection);
    }

    let job_id = projection.nextJobID & projection.jobIDMask;
    let mut result = ZSTDMT_createJobResult {
        action: CREATE_JOB_POST,
        jobID: job_id,
        jobNumber: projection.nextJobID,
        nextJobID: projection.nextJobID,
        jobReady: projection.jobReady,
        ..ZSTDMT_createJobResult::default()
    };

    if projection.jobReady == 0 {
        debug_assert!(projection.inBuffFilled >= projection.srcSize);
        let (_, initialization) = create_job_initialization(projection);
        prepare_job(job_id, &initialization);

        /* A non-first empty job is represented by the terminal empty block,
         * not by a worker submission. */
        if projection.srcSize == 0 && projection.nextJobID > 0 {
            write_empty_job(job_id);
            result.action = CREATE_JOB_EMPTY;
            result.nextJobID = projection.nextJobID.wrapping_add(1);
            result.jobReady = 0;
            return result;
        }
    }

    if try_add_job(job_id) {
        result.nextJobID = projection.nextJobID.wrapping_add(1);
        result.jobReady = 0;
    } else {
        result.jobReady = 1;
    }
    result
}

/// Apply MT scalar scheduling policy while C retains all private descriptor,
/// pool, mutex, condition-variable, and worker-callback operations.
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_createCompressionJob(
    projection: *const ZSTDMT_createJobProjection,
    opaque: *mut c_void,
    prepareJob: Option<ZSTDMT_prepareJobFn>,
    writeEmptyJob: Option<ZSTDMT_writeEmptyJobFn>,
    tryAddJob: Option<ZSTDMT_tryAddJobFn>,
) -> ZSTDMT_createJobResult {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ZSTDMT_createJobResult::default();
    };
    let (Some(prepare_job), Some(write_empty_job), Some(try_add_job)) =
        (prepareJob, writeEmptyJob, tryAddJob)
    else {
        return ZSTDMT_createJobResult::default();
    };

    create_compression_job_with(
        projection,
        |job_id, initialization| unsafe {
            prepare_job(opaque, job_id, initialization);
        },
        |job_id| unsafe {
            write_empty_job(opaque, job_id);
        },
        |job_id| unsafe { try_add_job(opaque, job_id) != 0 },
    )
}

#[inline]
unsafe fn publish_finish_error(
    publication: ZSTDMT_compressionJobFinishPublicationProjection,
    error: usize,
) {
    let (Some(lock), Some(unlock)) = (publication.lock, publication.unlock) else {
        return;
    };
    if publication.cSize.is_null() {
        return;
    }
    unsafe {
        lock(publication.callbackContext);
        *publication.cSize = error;
        unlock(publication.callbackContext);
    }
}

#[inline]
unsafe fn publish_finish_last_block_size(
    publication: ZSTDMT_compressionJobFinishPublicationProjection,
    last_block_size: usize,
) {
    let (Some(lock), Some(unlock)) = (publication.lock, publication.unlock) else {
        return;
    };
    if publication.cSize.is_null() {
        return;
    }
    unsafe {
        lock(publication.callbackContext);
        *publication.cSize = (*publication.cSize).wrapping_add(last_block_size);
        unlock(publication.callbackContext);
    }
}

#[inline]
unsafe fn publish_finish_consumed(
    publication: ZSTDMT_compressionJobFinishPublicationProjection,
    src_size: usize,
) {
    debug_assert_eq!(publication.srcSize, src_size);
    if let Some(validate_consumed) = publication.validateConsumed {
        unsafe { validate_consumed(publication.callbackContext, src_size) };
    }
    let (Some(lock), Some(unlock)) = (publication.lock, publication.unlock) else {
        return;
    };
    if publication.consumed.is_null() {
        return;
    }
    unsafe {
        lock(publication.callbackContext);
        *publication.consumed = src_size;
        unlock(publication.callbackContext);
    }
}

#[inline]
unsafe fn signal_finish(publication: ZSTDMT_compressionJobFinishPublicationProjection) {
    let (Some(lock), Some(signal), Some(unlock)) =
        (publication.lock, publication.signal, publication.unlock)
    else {
        return;
    };
    unsafe {
        lock(publication.callbackContext);
        signal(publication.callbackContext);
        unlock(publication.callbackContext);
    }
}

/// Finish a worker job with Rust-owned branch and callback ordering.
///
/// Error publication happens before serial completion and resource release,
/// matching the original C path so failed jobs advance the serial state with
/// the published error. Success publishes the final block size only after the
/// private sequence and CCtx callbacks return. Both paths publish the full
/// consumed size and signal only after all preceding leaves have run.
#[inline]
fn finish_compression_job_with<E, F, R, C, B, P, S>(
    error: usize,
    last_block_size: usize,
    src_size: usize,
    mut publish_error: E,
    mut ensure_finished: F,
    mut release_seq: R,
    mut release_cctx: C,
    mut publish_last_block_size: B,
    mut publish_consumed: P,
    mut signal: S,
) where
    E: FnMut(usize),
    F: FnMut(),
    R: FnMut(),
    C: FnMut(),
    B: FnMut(usize),
    P: FnMut(usize),
    S: FnMut(),
{
    let failed = ERR_isError(error);
    if failed {
        publish_error(error);
    }
    ensure_finished();
    release_seq();
    release_cctx();
    if !failed {
        publish_last_block_size(last_block_size);
    }
    publish_consumed(src_size);
    signal();
}

/// Run the high-level worker-job sequence while C owns all codec and
/// synchronization operations behind callbacks.
///
/// Resource acquisition and every codec-facing stage can fail with a zstd
/// error. Rust stops at the first such error, normalizes the final block size,
/// then applies the C-owned finish leaves through the Rust-owned policy above.
#[inline]
fn compression_job_with<A, P, S, B, Q, H, C, T>(
    projection: ZSTDMT_compressionJobProjection,
    finish: ZSTDMT_compressionJobFinishProjection,
    mut acquire_resources: A,
    mut prepare_parameters: P,
    mut generate_sequences: S,
    mut begin_job: B,
    mut apply_sequences: Q,
    mut write_frame_header: H,
    mut compress_job: C,
    mut trace_job: T,
) where
    A: FnMut() -> usize,
    P: FnMut(),
    S: FnMut(),
    B: FnMut() -> usize,
    Q: FnMut(),
    H: FnMut() -> usize,
    C: FnMut(c_uint) -> ZSTDMT_chunkProcessResult,
    T: FnMut(),
{
    let (Some(ensure_finished), Some(release_seq), Some(release_cctx)) = (
        finish.ensureFinished,
        finish.releaseSeq,
        finish.releaseCCtx,
    )
    else {
        return;
    };

    let mut error = acquire_resources();
    let mut last_block_size = 0;

    if !ERR_isError(error) {
        prepare_parameters();
        generate_sequences();
        error = begin_job();
    }

    if !ERR_isError(error) {
        apply_sequences();
        if projection.firstJob == 0 {
            error = write_frame_header();
        }
    }

    if !ERR_isError(error) {
        let result = compress_job(projection.lastJob);
        if ERR_isError(result.error) {
            error = result.error;
        } else {
            last_block_size = result.lastBlockSize;
            trace_job();
        }
    }

    if ERR_isError(error) {
        last_block_size = 0;
    }
    let publication = finish.publication;
    finish_compression_job_with(
        error,
        last_block_size,
        finish.srcSize,
        |error| unsafe { publish_finish_error(publication, error) },
        || unsafe { ensure_finished(finish.callbackContext) },
        || unsafe { release_seq(finish.callbackContext) },
        || unsafe { release_cctx(finish.callbackContext) },
        |last_block_size| unsafe { publish_finish_last_block_size(publication, last_block_size) },
        |src_size| unsafe { publish_finish_consumed(publication, src_size) },
        || unsafe { signal_finish(publication) },
    );
}

/// Drive one ordered MT serial-state turn.
///
/// The wait callback returns with the C-owned serial mutex held.  Rust then
/// performs the LDM-before-checksum policy only for the current job and always
/// invokes the advance callback exactly once, including when an earlier job
/// has already skipped this turn.
#[inline]
fn serial_state_gen_sequences_with<W, U, G, P, C, A>(
    seq_store: &mut ZstdMtRawSeqStore,
    src: *const c_void,
    src_size: usize,
    job_id: c_uint,
    ldm_enabled: bool,
    checksum_enabled: bool,
    mut wait_for_turn: W,
    mut update_ldm_window: U,
    mut generate_ldm_sequences: G,
    mut publish_ldm_window: P,
    mut update_checksum: C,
    mut advance: A,
) where
    W: FnMut(c_uint) -> bool,
    U: FnMut(&mut ZstdMtRawSeqStore, *const c_void, usize),
    G: FnMut(&mut ZstdMtRawSeqStore, *const c_void, usize),
    P: FnMut(),
    C: FnMut(*const c_void, usize),
    A: FnMut(),
{
    if wait_for_turn(job_id) {
        if ldm_enabled {
            update_ldm_window(seq_store, src, src_size);
            generate_ldm_sequences(seq_store, src, src_size);
            publish_ldm_window();
        }
        if checksum_enabled && src_size != 0 {
            update_checksum(src, src_size);
        }
    }
    advance();
}

/// Own MT serial turn/skip and LDM/checksum ordering while C retains the
/// mutexes, LDM window/hash state, checksum state, and raw sequence storage.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateGenSequences(
    seq_store: *mut ZstdMtRawSeqStore,
    src: *const c_void,
    src_size: usize,
    job_id: c_uint,
    ldm_enabled: c_int,
    checksum_enabled: c_int,
    opaque: *mut c_void,
    wait_for_turn: Option<ZSTDMT_serialWaitForTurnFn>,
    update_ldm_window: Option<ZSTDMT_serialLdmWindowUpdateFn>,
    generate_ldm_sequences: Option<ZSTDMT_serialLdmGenerateSequencesFn>,
    publish_ldm_window: Option<ZSTDMT_serialLdmPublishWindowFn>,
    update_checksum: Option<ZSTDMT_serialUpdateChecksumFn>,
    advance: Option<ZSTDMT_serialAdvanceFn>,
) {
    let (
        Some(wait_for_turn),
        Some(update_ldm_window),
        Some(generate_ldm_sequences),
        Some(publish_ldm_window),
        Some(update_checksum),
        Some(advance),
    ) = (
        wait_for_turn,
        update_ldm_window,
        generate_ldm_sequences,
        publish_ldm_window,
        update_checksum,
        advance,
    )
    else {
        return;
    };
    if seq_store.is_null() {
        return;
    }

    unsafe {
        serial_state_gen_sequences_with(
            &mut *seq_store,
            src,
            src_size,
            job_id,
            ldm_enabled != 0,
            checksum_enabled != 0,
            |job_id| wait_for_turn(opaque, job_id) != 0,
            |seq_store, src, src_size| update_ldm_window(opaque, seq_store, src, src_size),
            |seq_store, src, src_size| {
                generate_ldm_sequences(opaque, seq_store, src, src_size)
            },
            || publish_ldm_window(opaque),
            |src, src_size| update_checksum(opaque, src, src_size),
            || advance(opaque),
        );
    }
}

/// Decide whether a failed job must advance the shared serial turn.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_serialStateEnsureFinished(
    next_job_id: c_uint,
    job_id: c_uint,
) -> ZSTDMT_serialStateEnsureFinishedResult {
    if next_job_id <= job_id {
        ZSTDMT_serialStateEnsureFinishedResult {
            skip: 1,
            nextJobID: job_id.wrapping_add(1),
        }
    } else {
        ZSTDMT_serialStateEnsureFinishedResult {
            skip: 0,
            nextJobID: next_job_id,
        }
    }
}

/// Own the failed-job serial cleanup order while C retains synchronization and
/// private LDM-window operations behind callbacks.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateEnsureFinishedOrchestrated(
    state: *const ZSTDMT_RustSerialEnsureFinishedState,
) {
    if state.is_null() {
        return;
    }
    let state = unsafe { &*state };
    if state.callback_context.is_null() || state.next_job_id.is_null() {
        return;
    }
    let (
        Some(lock),
        Some(broadcast),
        Some(ldm_lock),
        Some(clear_ldm_window),
        Some(ldm_signal),
        Some(ldm_unlock),
        Some(unlock),
        Some(on_skip),
    ) = (
        state.lock,
        state.broadcast,
        state.ldm_lock,
        state.clear_ldm_window,
        state.ldm_signal,
        state.ldm_unlock,
        state.unlock,
        state.on_skip,
    )
    else {
        return;
    };

    unsafe {
        lock(state.callback_context);
        let result = ZSTDMT_rust_serialStateEnsureFinished(*state.next_job_id, state.job_id);
        if result.skip != 0 {
            on_skip(state.callback_context, state.job_id, state.c_size);
            *state.next_job_id = result.nextJobID;
            broadcast(state.callback_context);
            ldm_lock(state.callback_context);
            clear_ldm_window(state.callback_context);
            ldm_signal(state.callback_context);
            ldm_unlock(state.callback_context);
        }
        unlock(state.callback_context);
    }
}

/// Wait for each submitted MT job in ring order.  C retains the per-job
/// mutex, condition variable, and consumed/source counters behind one
/// callback; Rust owns the ring-slot and completion progression policy.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_waitForAllJobsCompleted(
    mut done_job_id: c_uint,
    next_job_id: c_uint,
    job_id_mask: c_uint,
    opaque: *mut c_void,
    wait_for_job: Option<ZSTDMT_waitForJobCompleteFn>,
) -> c_uint {
    let Some(wait_for_job) = wait_for_job else {
        return done_job_id;
    };
    while done_job_id < next_job_id {
        unsafe { wait_for_job(opaque, done_job_id & job_id_mask, done_job_id) };
        done_job_id = done_job_id.wrapping_add(1);
    }
    done_job_id
}

/// Visit every MT job slot so C can release its private descriptor resources.
/// Rust owns the deterministic inclusive ring-table iteration; C retains the
/// buffer-pool, mutex/condition, and descriptor cleanup side effects.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_releaseAllJobResources(
    job_id_mask: c_uint,
    opaque: *mut c_void,
    release_job: Option<ZSTDMT_releaseJobResourceFn>,
) {
    let Some(release_job) = release_job else {
        return;
    };
    for job_id in 0..=job_id_mask {
        unsafe { release_job(opaque, job_id) };
    }
}

/// C ABI entry point for the worker-job orchestration. C supplies callbacks
/// that keep the private descriptor, pools, mutexes, and codec operations on
/// the C side of this narrow projection.
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressionJob(
    projection: *const ZSTDMT_compressionJobProjection,
    finishProjection: *const ZSTDMT_compressionJobFinishProjection,
    opaque: *mut c_void,
    acquireResources: Option<ZSTDMT_compressionJobStepFn>,
    prepareParameters: Option<ZSTDMT_compressionJobVoidFn>,
    generateSequences: Option<ZSTDMT_compressionJobVoidFn>,
    beginJob: Option<ZSTDMT_compressionJobStepFn>,
    applySequences: Option<ZSTDMT_compressionJobApplySequencesFn>,
    compressJob: Option<ZSTDMT_compressionJobCompressFn>,
    traceJob: Option<ZSTDMT_compressionJobVoidFn>,
) {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return;
    };
    let Some(finish) = (unsafe { finishProjection.as_ref() }).copied() else {
        return;
    };
    let (
        Some(acquire_resources),
        Some(prepare_parameters),
        Some(generate_sequences),
        Some(begin_job),
        Some(apply_sequences),
        Some(compress_job),
        Some(trace_job),
    ) = (
        acquireResources,
        prepareParameters,
        generateSequences,
        beginJob,
        applySequences,
        compressJob,
        traceJob,
    )
    else {
        return;
    };

    let sequence_state = projection.sequenceState;

    compression_job_with(
        projection,
        finish,
        || unsafe { acquire_resources(opaque) },
        || unsafe { prepare_parameters(opaque) },
        || unsafe { generate_sequences(opaque) },
        || unsafe { begin_job(opaque) },
        || {
            let sequence_projection =
                unsafe { compression_job_sequence_projection(sequence_state) };
            apply_compression_job_sequences_with(sequence_projection, |cctx, sequences, count| {
                unsafe { apply_sequences(opaque, cctx, sequences, count) }
            });
        },
        || unsafe { ZSTDMT_rust_writeFrameHeader(projection.frameHeaderState) },
        |last_job| unsafe { compress_job(opaque, last_job) },
        || unsafe { trace_job(opaque) },
    );
}

#[inline]
fn normalize_mt_job_size(job_size: usize, job_size_min: usize, job_size_max: usize) -> usize {
    if job_size != 0 && job_size < job_size_min {
        job_size_min
    } else if job_size > job_size_max {
        job_size_max
    } else {
        job_size
    }
}

#[inline]
fn init_serial_reset_result(status: c_int) -> usize {
    if status != 0 {
        ERROR(ZstdErrorCode::MemoryAllocation)
    } else {
        0
    }
}

/// Map the private C serial-reset result to the MT stream-initialization ABI
/// error. C retains serial-state allocation and mutation; Rust owns only this
/// scalar error policy.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_initSerialResetResult(status: c_int) -> usize {
    init_serial_reset_result(status)
}

#[inline]
fn init_round_buffer_capacity(
    projection: ZSTDMT_initCStreamProjection,
    target_prefix_size: usize,
    target_section_size: usize,
) -> usize {
    let window_size = if projection.enableLdm == ZSTD_PS_ENABLE {
        1usize << projection.windowLog as usize
    } else {
        0
    };
    let nb_slack_buffers = 2 + usize::from(target_prefix_size > 0);
    let slack_size = target_section_size.wrapping_mul(nb_slack_buffers);
    let nb_workers = projection.requestedNbWorkers.max(1) as usize;
    let sections_size = target_section_size.wrapping_mul(nb_workers);
    window_size.max(sections_size).wrapping_add(slack_size)
}

#[inline]
fn init_rsync_parameters(target_section_size: usize) -> (u64, u64) {
    let job_size_kb = (target_section_size >> 10) as u32;
    debug_assert!(job_size_kb >= 1);
    let rsync_bits = ZSTD_highbit32(job_size_kb) + 10;
    debug_assert!(rsync_bits >= (RSYNC_MIN_BLOCK_LOG + 2) as u32);
    let hit_mask = 1u64.wrapping_shl(rsync_bits).wrapping_sub(1);
    (hit_mask, rolling_hash_prime_power(RSYNC_LENGTH as c_uint))
}

/// Run the high-level MT streaming initialization policy while C retains all
/// private state and side effects behind callbacks.  The callback order is
/// intentionally the order of the original C initializer: resize, normalize,
/// wait for all jobs, release their resources, attach the first dictionary,
/// size buffers, reset stream state,
/// update the active dictionary, and finally reset serial state.
#[inline]
fn init_c_stream_with<
    Resize,
    WaitForAllJobs,
    ReleaseAllJobResources,
    ApplyParams,
    PrepareDictionary,
    SetPrefixSize,
    SetSectionSize,
    SetRsync,
    SetBufferSize,
    ResizeRoundBuffer,
    ResetStream,
    UpdateDictionary,
    SerialReset,
>(
    projection: ZSTDMT_initCStreamProjection,
    mut resize: Resize,
    mut wait_for_all_jobs: WaitForAllJobs,
    mut release_all_job_resources: ReleaseAllJobResources,
    mut apply_parameters: ApplyParams,
    mut prepare_dictionary: PrepareDictionary,
    mut set_target_prefix_size: SetPrefixSize,
    mut set_target_section_size: SetSectionSize,
    mut set_rsync: SetRsync,
    mut set_buffer_size: SetBufferSize,
    mut resize_round_buffer: ResizeRoundBuffer,
    mut reset_stream: ResetStream,
    mut update_dictionary: UpdateDictionary,
    mut serial_reset: SerialReset,
) -> usize
where
    Resize: FnMut(c_uint) -> usize,
    WaitForAllJobs: FnMut(),
    ReleaseAllJobResources: FnMut(),
    ApplyParams: FnMut(usize),
    PrepareDictionary: FnMut() -> usize,
    SetPrefixSize: FnMut(usize),
    SetSectionSize: FnMut(usize),
    SetRsync: FnMut(u64, u64),
    SetBufferSize: FnMut(usize),
    ResizeRoundBuffer: FnMut(usize) -> usize,
    ResetStream: FnMut(),
    UpdateDictionary: FnMut() -> usize,
    SerialReset: FnMut(usize) -> usize,
{
    if projection.requestedNbWorkers != projection.currentNbWorkers {
        let error = resize(projection.requestedNbWorkers);
        if ERR_isError(error) {
            return error;
        }
    }

    let job_size = normalize_mt_job_size(
        projection.jobSize,
        projection.jobSizeMin,
        projection.jobSizeMax,
    );

    if projection.allJobsCompleted == 0 {
        wait_for_all_jobs();
        release_all_job_resources();
    }

    apply_parameters(job_size);
    let error = prepare_dictionary();
    if ERR_isError(error) {
        return error;
    }

    let target_prefix_size = compute_overlap_size(
        projection.windowLog,
        projection.chainLog,
        projection.strategy,
        projection.overlapLog,
        projection.enableLdm,
    );
    set_target_prefix_size(target_prefix_size);

    let initial_target_section_size = if job_size == 0 {
        1usize
            << compute_target_job_log(
                projection.windowLog,
                projection.chainLog,
                projection.strategy,
                projection.enableLdm,
            ) as usize
    } else {
        job_size
    };
    debug_assert!(initial_target_section_size <= projection.jobSizeMax);
    set_target_section_size(initial_target_section_size);

    if projection.rsyncable != 0 {
        let (hit_mask, prime_power) = init_rsync_parameters(initial_target_section_size);
        set_rsync(hit_mask, prime_power);
    }

    let target_section_size = initial_target_section_size.max(target_prefix_size);
    if target_section_size != initial_target_section_size {
        set_target_section_size(target_section_size);
    }

    set_buffer_size(crate::zstd_compress_api::ZSTD_compressBound(
        target_section_size,
    ));

    let round_capacity =
        init_round_buffer_capacity(projection, target_prefix_size, target_section_size);
    if projection.roundBuffCapacity < round_capacity {
        let error = resize_round_buffer(round_capacity);
        if ERR_isError(error) {
            return error;
        }
    }

    reset_stream();

    let error = update_dictionary();
    if ERR_isError(error) {
        return error;
    }

    serial_reset(target_section_size)
}

/// Run the MT serial-state reset policy while C retains its private LDM
/// tables, dictionary/window state, checksum state, and allocator callbacks.
/// Rust prepares the scalar LDM policy and table sizes, then the callback order
/// mirrors the original reset path and stops immediately when table allocation
/// reports failure.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct ZSTDMT_serialResetPlan {
    ldmParams: ZSTDMT_serialResetLdmParameters,
    maxNbSeq: usize,
    hashSize: usize,
    numBuckets: usize,
    bucketLog: c_uint,
    previousBucketLog: c_uint,
}

#[inline]
unsafe fn serial_state_reset_plan(
    projection: ZSTDMT_RustSerialResetProjection,
) -> ZSTDMT_serialResetPlan {
    let mut ldm_params = ZSTDMT_serialResetLdmParameters {
        enableLdm: projection.enableLdm,
        hashLog: projection.hashLog,
        bucketSizeLog: projection.bucketSizeLog,
        minMatchLength: projection.minMatchLength,
        hashRateLog: projection.hashRateLog,
        windowLog: projection.windowLog,
    };
    if projection.enableLdm == ZSTD_PS_ENABLE {
        let c_params = ZSTD_compressionParameters {
            windowLog: projection.windowLog,
            strategy: projection.strategy,
            ..Default::default()
        };
        unsafe { ZSTD_ldm_adjustParameters(&mut ldm_params, &c_params) };
    } else {
        ldm_params = ZSTDMT_serialResetLdmParameters::default();
    }

    let ldm_enabled = ldm_params.enableLdm == ZSTD_PS_ENABLE;
    let bucket_log = if ldm_enabled {
        ldm_params.hashLog.wrapping_sub(ldm_params.bucketSizeLog)
    } else {
        0
    };
    let previous_bucket_log = if ldm_enabled {
        projection
            .previousHashLog
            .wrapping_sub(projection.previousBucketSizeLog)
    } else {
        0
    };
    let hash_size = if ldm_enabled {
        (1usize.wrapping_shl(ldm_params.hashLog)).wrapping_mul(projection.ldmEntrySize)
    } else {
        0
    };
    let num_buckets = if ldm_enabled {
        1usize.wrapping_shl(bucket_log)
    } else {
        0
    };
    let max_nb_seq = unsafe { ZSTD_ldm_getMaxNbSeq(ldm_params, projection.jobSize) };

    ZSTDMT_serialResetPlan {
        ldmParams: ldm_params,
        maxNbSeq: max_nb_seq,
        hashSize: hash_size,
        numBuckets: num_buckets,
        bucketLog: bucket_log,
        previousBucketLog: previous_bucket_log,
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateReset(
    projection: *const ZSTDMT_RustSerialResetProjection,
    opaque: *mut c_void,
    set_ldm_params: Option<ZSTDMT_serialResetSetLdmParamsFn>,
    reset_next_job: Option<ZSTDMT_serialResetVoidFn>,
    reset_checksum: Option<ZSTDMT_serialResetVoidFn>,
    set_nb_seq: Option<ZSTDMT_serialResetSetNbSeqFn>,
    reset_window: Option<ZSTDMT_serialResetVoidFn>,
    resize_tables: Option<ZSTDMT_serialResetResizeFn>,
    zero_tables: Option<ZSTDMT_serialResetZeroTablesFn>,
    load_dictionary: Option<ZSTDMT_serialResetVoidFn>,
    copy_window: Option<ZSTDMT_serialResetVoidFn>,
    publish_params: Option<ZSTDMT_serialResetPublishFn>,
) -> c_int {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return 1;
    };
    let (
        Some(set_ldm_params),
        Some(reset_next_job),
        Some(reset_checksum),
        Some(set_nb_seq),
        Some(reset_window),
        Some(resize_tables),
        Some(zero_tables),
        Some(load_dictionary),
        Some(copy_window),
        Some(publish_params),
    ) = (
        set_ldm_params,
        reset_next_job,
        reset_checksum,
        set_nb_seq,
        reset_window,
        resize_tables,
        zero_tables,
        load_dictionary,
        copy_window,
        publish_params,
    )
    else {
        return 1;
    };

    let plan = unsafe { serial_state_reset_plan(projection) };
    unsafe {
        set_ldm_params(
            opaque,
            plan.ldmParams.enableLdm,
            plan.ldmParams.hashLog,
            plan.ldmParams.bucketSizeLog,
            plan.ldmParams.minMatchLength,
            plan.ldmParams.hashRateLog,
            plan.ldmParams.windowLog,
        );
        reset_next_job(opaque);
        if projection.checksumEnabled != 0 {
            reset_checksum(opaque);
        }
        if plan.ldmParams.enableLdm == ZSTD_PS_ENABLE {
            set_nb_seq(opaque, plan.maxNbSeq);
            reset_window(opaque);
            if resize_tables(
                opaque,
                plan.hashSize,
                plan.numBuckets,
                plan.bucketLog,
                plan.previousBucketLog,
            ) != 0
            {
                return 1;
            }
            zero_tables(opaque, plan.hashSize, plan.numBuckets);
            load_dictionary(opaque);
            copy_window(opaque);
        }
        publish_params(opaque, projection.jobSize);
    }
    0
}

/// Run the MT serial-state synchronization initialization order and preserve
/// the original bitwise error aggregation. C owns each pthread primitive and
/// the zeroing callback.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateInit(
    opaque: *mut c_void,
    zero_state: Option<ZSTDMT_serialStateVoidFn>,
    init_mutex: Option<ZSTDMT_serialStateInitFn>,
    init_cond: Option<ZSTDMT_serialStateInitFn>,
    init_ldm_mutex: Option<ZSTDMT_serialStateInitFn>,
    init_ldm_cond: Option<ZSTDMT_serialStateInitFn>,
) -> c_int {
    let (
        Some(zero_state),
        Some(init_mutex),
        Some(init_cond),
        Some(init_ldm_mutex),
        Some(init_ldm_cond),
    ) = (
        zero_state,
        init_mutex,
        init_cond,
        init_ldm_mutex,
        init_ldm_cond,
    )
    else {
        return 1;
    };

    unsafe {
        zero_state(opaque);
        init_mutex(opaque) | init_cond(opaque) | init_ldm_mutex(opaque) | init_ldm_cond(opaque)
    }
}

/// Run the MT serial-state synchronization destruction and table-free order.
/// The callbacks retain ownership of platform synchronization and allocator
/// details on the C side.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_serialStateFree(
    opaque: *mut c_void,
    destroy_mutex: Option<ZSTDMT_serialStateVoidFn>,
    destroy_cond: Option<ZSTDMT_serialStateVoidFn>,
    destroy_ldm_mutex: Option<ZSTDMT_serialStateVoidFn>,
    destroy_ldm_cond: Option<ZSTDMT_serialStateVoidFn>,
    free_tables: Option<ZSTDMT_serialStateVoidFn>,
) {
    let (
        Some(destroy_mutex),
        Some(destroy_cond),
        Some(destroy_ldm_mutex),
        Some(destroy_ldm_cond),
        Some(free_tables),
    ) = (
        destroy_mutex,
        destroy_cond,
        destroy_ldm_mutex,
        destroy_ldm_cond,
        free_tables,
    )
    else {
        return;
    };

    unsafe {
        destroy_mutex(opaque);
        destroy_cond(opaque);
        destroy_ldm_mutex(opaque);
        destroy_ldm_cond(opaque);
        free_tables(opaque);
    }
}

/// Run the MT context teardown order while C callbacks retain all private
/// layouts, allocators, worker pools, and synchronization details.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_freeCCtx(state: *const ZSTDMT_RustFreeCCtxState) -> usize {
    if state.is_null() {
        return 0;
    }
    let state = unsafe { &*state };
    let (
        Some(free_factory),
        Some(release_all_job_resources),
        Some(free_jobs),
        Some(free_buffer_pool),
        Some(free_cctx_pool),
        Some(free_seq_pool),
        Some(free_serial_state),
        Some(free_cdict),
        Some(free_round_buffer),
        Some(free_mtctx),
    ) = (
        state.free_factory,
        state.release_all_job_resources,
        state.free_jobs,
        state.free_buffer_pool,
        state.free_cctx_pool,
        state.free_seq_pool,
        state.free_serial_state,
        state.free_cdict,
        state.free_round_buffer,
        state.free_mtctx,
    )
    else {
        return 0;
    };

    unsafe {
        if state.provided_factory == 0 {
            free_factory(state.callback_context);
        }
        release_all_job_resources(state.callback_context);
        free_jobs(state.callback_context);
        free_buffer_pool(state.callback_context);
        free_cctx_pool(state.callback_context);
        free_seq_pool(state.callback_context);
        free_serial_state(state.callback_context);
        free_cdict(state.callback_context);
        if state.round_buffer_present != 0 {
            free_round_buffer(state.callback_context);
        }
        free_mtctx(state.callback_context);
    }
    0
}

/// C ABI entry point for the MT streaming initializer.  C owns every
/// allocation, dictionary handle, synchronization object, and private context
/// mutation; this wrapper only connects those operations to the Rust policy.
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_initCStream(
    projection: *const ZSTDMT_initCStreamProjection,
    opaque: *mut c_void,
    resize: Option<ZSTDMT_initResizeFn>,
    waitForAllJobs: Option<ZSTDMT_initWaitForAllJobsFn>,
    releaseAllJobResources: Option<ZSTDMT_initReleaseAllJobResourcesFn>,
    applyParameters: Option<ZSTDMT_initApplyParametersFn>,
    prepareDictionary: Option<ZSTDMT_initDictionaryFn>,
    setTargetPrefixSize: Option<ZSTDMT_initSetSizeFn>,
    setTargetSectionSize: Option<ZSTDMT_initSetSizeFn>,
    setRsync: Option<ZSTDMT_initSetRsyncFn>,
    setBufferSize: Option<ZSTDMT_initSetBufferSizeFn>,
    resizeRoundBuffer: Option<ZSTDMT_initResizeRoundBufferFn>,
    resetStream: Option<ZSTDMT_initResetStreamFn>,
    updateDictionary: Option<ZSTDMT_initDictionaryFn>,
    serialReset: Option<ZSTDMT_initSerialResetFn>,
) -> usize {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ERROR(ZstdErrorCode::Generic);
    };
    let (
        Some(resize),
        Some(wait_for_all_jobs),
        Some(release_all_job_resources),
        Some(apply_parameters),
        Some(prepare_dictionary),
        Some(set_target_prefix_size),
        Some(set_target_section_size),
        Some(set_rsync),
        Some(set_buffer_size),
        Some(resize_round_buffer),
        Some(reset_stream),
        Some(update_dictionary),
        Some(serial_reset),
    ) = (
        resize,
        waitForAllJobs,
        releaseAllJobResources,
        applyParameters,
        prepareDictionary,
        setTargetPrefixSize,
        setTargetSectionSize,
        setRsync,
        setBufferSize,
        resizeRoundBuffer,
        resetStream,
        updateDictionary,
        serialReset,
    )
    else {
        return ERROR(ZstdErrorCode::Generic);
    };

    init_c_stream_with(
        projection,
        |nb_workers| unsafe { resize(opaque, nb_workers) },
        || unsafe { wait_for_all_jobs(opaque) },
        || unsafe { release_all_job_resources(opaque) },
        |job_size| unsafe { apply_parameters(opaque, job_size) },
        || unsafe { prepare_dictionary(opaque) },
        |size| unsafe { set_target_prefix_size(opaque, size) },
        |size| unsafe { set_target_section_size(opaque, size) },
        |hit_mask, prime_power| unsafe { set_rsync(opaque, hit_mask, prime_power) },
        |size| unsafe { set_buffer_size(opaque, size) },
        |capacity| unsafe { resize_round_buffer(opaque, capacity) },
        || unsafe { reset_stream(opaque) },
        || unsafe { update_dictionary(opaque) },
        |target_section_size| unsafe { serial_reset(opaque, target_section_size) },
    )
}

#[inline]
fn invalid_flush_publication(
    output_pos: usize,
    dst_flushed: usize,
) -> ZSTDMT_flushPublicationResult {
    ZSTDMT_flushPublicationResult {
        status: FLUSH_PUBLICATION_INVALID_BOUNDS,
        toFlush: 0,
        outputPos: output_pos,
        dstFlushed: dst_flushed,
    }
}

#[inline]
unsafe fn publish_job_output_with(
    output_dst: *mut c_void,
    output_size: usize,
    output_pos: usize,
    job_dst: *const c_void,
    job_capacity: usize,
    c_size: usize,
    dst_flushed: usize,
) -> ZSTDMT_flushPublicationResult {
    let Some(output_available) = output_size.checked_sub(output_pos) else {
        return invalid_flush_publication(output_pos, dst_flushed);
    };
    let Some(job_available) = c_size.checked_sub(dst_flushed) else {
        return invalid_flush_publication(output_pos, dst_flushed);
    };
    if c_size > job_capacity {
        return invalid_flush_publication(output_pos, dst_flushed);
    }

    let to_flush = output_available.min(job_available);
    if to_flush > 0 {
        if output_dst.is_null() || job_dst.is_null() {
            return invalid_flush_publication(output_pos, dst_flushed);
        }
        unsafe {
            ptr::copy_nonoverlapping(
                job_dst.cast::<u8>().add(dst_flushed),
                output_dst.cast::<u8>().add(output_pos),
                to_flush,
            );
        }
    }

    ZSTDMT_flushPublicationResult {
        status: FLUSH_PUBLICATION_OK,
        toFlush: to_flush,
        outputPos: output_pos + to_flush,
        dstFlushed: dst_flushed + to_flush,
    }
}

#[inline]
fn flush_produced_result(result: usize, output_pos: usize) -> ZSTDMT_flushProducedResult {
    ZSTDMT_flushProducedResult {
        result,
        outputPos: output_pos,
        updateAllJobsCompleted: 0,
        allJobsCompleted: 0,
    }
}

#[inline]
unsafe fn flush_produced_with<P, C, U, F, W, R>(
    context: ZSTDMT_flushContextProjection,
    output_dst: *mut c_void,
    output_size: usize,
    output_pos: usize,
    block_to_flush: c_uint,
    end: c_uint,
    mut project_job: P,
    mut add_frame_checksum: C,
    mut update_job: U,
    mut complete_job: F,
    mut wait_for_all_jobs: W,
    mut release_all_job_resources: R,
) -> ZSTDMT_flushProducedResult
where
    P: FnMut(c_uint, c_uint) -> ZSTDMT_flushJobProjection,
    C: FnMut(c_uint, &mut ZSTDMT_flushJobProjection),
    U: FnMut(c_uint, usize),
    F: FnMut(c_uint, usize, usize),
    W: FnMut(),
    R: FnMut(),
{
    let mut output_pos = output_pos;
    let mut done_job_id = context.doneJobID;

    if done_job_id < context.nextJobID {
        let job_id = done_job_id & context.jobIDMask;
        let mut projection = project_job(job_id, block_to_flush);
        let src_consumed = projection.consumed;
        let src_size = projection.srcSize;

        if ERR_isError(projection.cSize) {
            wait_for_all_jobs();
            release_all_job_resources();
            return flush_produced_result(projection.cSize, output_pos);
        }

        /* The checksum is added only once, after the worker has consumed the
         * whole source range.  C retains the serial XXH state and updates the
         * projected cSize through this callback. */
        if src_consumed == src_size && projection.frameChecksumNeeded != 0 {
            add_frame_checksum(job_id, &mut projection);
        }

        if ERR_isError(projection.cSize) {
            wait_for_all_jobs();
            release_all_job_resources();
            return flush_produced_result(projection.cSize, output_pos);
        }

        let c_size = projection.cSize;
        let mut dst_flushed = projection.dstFlushed;
        if c_size > 0 {
            let publication = unsafe {
                publish_job_output_with(
                    output_dst,
                    output_size,
                    output_pos,
                    projection.dstStart,
                    projection.dstCapacity,
                    c_size,
                    dst_flushed,
                )
            };
            if publication.status != FLUSH_PUBLICATION_OK {
                return flush_produced_result(ERROR(ZstdErrorCode::Generic), output_pos);
            }

            output_pos = publication.outputPos;
            dst_flushed = publication.dstFlushed;
            update_job(job_id, dst_flushed);

            if src_consumed == src_size && dst_flushed == c_size {
                complete_job(job_id, src_size, c_size);
                done_job_id = done_job_id.wrapping_add(1);
            }
        }

        /* Match the C API's distinction between known pending output and a
         * live worker whose output size is not known yet. */
        if c_size > dst_flushed {
            return flush_produced_result(c_size - dst_flushed, output_pos);
        }
        if src_size > src_consumed {
            return flush_produced_result(1, output_pos);
        }
    }

    if done_job_id < context.nextJobID {
        return flush_produced_result(1, output_pos);
    }
    if context.jobReady != 0 {
        return flush_produced_result(1, output_pos);
    }
    if context.inBuffFilled > 0 {
        return flush_produced_result(1, output_pos);
    }

    /* All jobs are entirely flushed.  For ZSTD_e_end the caller asks whether
     * the frame itself has ended, rather than whether buffers are empty. */
    ZSTDMT_flushProducedResult {
        result: if end == ZSTD_E_END {
            (context.frameEnded == 0) as usize
        } else {
            0
        },
        outputPos: output_pos,
        updateAllJobsCompleted: 1,
        allJobsCompleted: context.frameEnded,
    }
}

#[cfg(not(test))]
unsafe extern "C" {
    fn ZSTD_compressContinue_public(
        cctx: *mut c_void,
        dst: *mut c_void,
        dst_capacity: usize,
        src: *const c_void,
        src_size: usize,
    ) -> usize;
    fn ZSTD_compressEnd_public(
        cctx: *mut c_void,
        dst: *mut c_void,
        dst_capacity: usize,
        src: *const c_void,
        src_size: usize,
    ) -> usize;
}

#[inline]
unsafe fn compress_job_chunks_with(
    cctx: *mut c_void,
    src: *const c_void,
    src_size: usize,
    dst: *mut c_void,
    dst_capacity: usize,
    chunk_size: usize,
    last_job: c_uint,
    progress_context: *mut c_void,
    progress_callback: Option<ZSTDMT_chunkProgressFn>,
    compress_continue: ZstdMtCompressFn,
    compress_end: ZstdMtCompressFn,
) -> ZSTDMT_chunkProcessResult {
    debug_assert!(chunk_size > 0);
    debug_assert!(chunk_size.is_power_of_two());

    let nb_chunks = src_size.div_ceil(chunk_size);
    let mut input = src.cast::<u8>();
    let mut produced = 0usize;

    for chunk_number in 1..nb_chunks {
        let c_size = unsafe {
            compress_continue(
                cctx,
                dst.cast::<u8>().wrapping_add(produced).cast(),
                dst_capacity.wrapping_sub(produced),
                input.cast(),
                chunk_size,
            )
        };
        if ERR_isError(c_size) {
            return ZSTDMT_chunkProcessResult {
                error: c_size,
                lastBlockSize: 0,
            };
        }
        input = input.wrapping_add(chunk_size);
        produced = produced.wrapping_add(c_size);
        debug_assert!(produced < dst_capacity);

        if let Some(callback) = progress_callback {
            unsafe { callback(progress_context, c_size, chunk_size * chunk_number) };
        }
    }

    if nb_chunks > 0 || last_job != 0 {
        let last_block_size1 = src_size & (chunk_size - 1);
        let last_block_size = if last_block_size1 == 0 && src_size >= chunk_size {
            chunk_size
        } else {
            last_block_size1
        };
        let c_size = unsafe {
            let dst = dst.cast::<u8>().wrapping_add(produced).cast();
            let dst_capacity = dst_capacity.wrapping_sub(produced);
            let compress = if last_job != 0 {
                compress_end
            } else {
                compress_continue
            };
            compress(cctx, dst, dst_capacity, input.cast(), last_block_size)
        };
        if ERR_isError(c_size) {
            return ZSTDMT_chunkProcessResult {
                error: c_size,
                lastBlockSize: 0,
            };
        }
        return ZSTDMT_chunkProcessResult {
            error: 0,
            lastBlockSize: c_size,
        };
    }

    ZSTDMT_chunkProcessResult {
        error: 0,
        lastBlockSize: 0,
    }
}

#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressJobChunks(
    cctx: *mut c_void,
    src: *const c_void,
    src_size: usize,
    dst: *mut c_void,
    dst_capacity: usize,
    chunk_size: usize,
    last_job: c_uint,
    progress_context: *mut c_void,
    progress_callback: Option<ZSTDMT_chunkProgressFn>,
) -> ZSTDMT_chunkProcessResult {
    unsafe {
        compress_job_chunks_with(
            cctx,
            src,
            src_size,
            dst,
            dst_capacity,
            chunk_size,
            last_job,
            progress_context,
            progress_callback,
            ZSTD_compressContinue_public,
            ZSTD_compressEnd_public,
        )
    }
}

#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_publishJobOutput(
    output_dst: *mut c_void,
    output_size: usize,
    output_pos: usize,
    job_dst: *const c_void,
    job_capacity: usize,
    c_size: usize,
    dst_flushed: usize,
) -> ZSTDMT_flushPublicationResult {
    unsafe {
        publish_job_output_with(
            output_dst,
            output_size,
            output_pos,
            job_dst,
            job_capacity,
            c_size,
            dst_flushed,
        )
    }
}

#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_flushProduced(
    context: *const ZSTDMT_flushContextProjection,
    output_dst: *mut c_void,
    output_size: usize,
    output_pos: usize,
    block_to_flush: c_uint,
    end: c_uint,
    opaque: *mut c_void,
    projectJob: Option<ZSTDMT_flushProjectJobFn>,
    addFrameChecksum: Option<ZSTDMT_flushChecksumFn>,
    updateJob: Option<ZSTDMT_flushUpdateJobFn>,
    completeJob: Option<ZSTDMT_flushCompleteJobFn>,
    waitForAllJobs: Option<ZSTDMT_flushWaitForAllJobsFn>,
    releaseAllJobResources: Option<ZSTDMT_flushReleaseAllJobResourcesFn>,
) -> ZSTDMT_flushProducedResult {
    let Some(context) = (unsafe { context.as_ref() }).copied() else {
        return flush_produced_result(ERROR(ZstdErrorCode::Generic), output_pos);
    };
    let (
        Some(project_job),
        Some(add_frame_checksum),
        Some(update_job),
        Some(complete_job),
        Some(wait_for_all_jobs),
        Some(release_all_job_resources),
    ) = (
        projectJob,
        addFrameChecksum,
        updateJob,
        completeJob,
        waitForAllJobs,
        releaseAllJobResources,
    )
    else {
        return flush_produced_result(ERROR(ZstdErrorCode::Generic), output_pos);
    };

    unsafe {
        flush_produced_with(
            context,
            output_dst,
            output_size,
            output_pos,
            block_to_flush,
            end,
            |job_id, block_to_flush| {
                let mut projection = ZSTDMT_flushJobProjection::default();
                project_job(opaque, job_id, block_to_flush, &mut projection);
                projection
            },
            |job_id, projection| {
                add_frame_checksum(opaque, job_id, projection);
            },
            |job_id, dst_flushed| {
                update_job(opaque, job_id, dst_flushed);
            },
            |job_id, src_size, c_size| {
                complete_job(opaque, job_id, src_size, c_size);
            },
            || {
                wait_for_all_jobs(opaque);
            },
            || {
                release_all_job_resources(opaque);
            },
        )
    }
}

#[inline]
fn compress_stream_result(
    result: usize,
    input_pos: usize,
    output_pos: usize,
    in_buff_filled: usize,
) -> ZSTDMT_compressStreamResult {
    ZSTDMT_compressStreamResult {
        result,
        inputPos: input_pos,
        outputPos: output_pos,
        inBuffFilled: in_buff_filled,
    }
}

#[inline]
fn publishable_in_buff_filled(buffer_start: *mut c_void, buffer_filled: usize) -> usize {
    if buffer_start.is_null() {
        0
    } else {
        buffer_filled
    }
}

/// Run one outer MT streaming pass while C retains the context and worker
/// implementation details.
///
/// The callbacks are intentionally narrow: input-range acquisition exposes a
/// C-owned buffer, job creation can post the prepared job, and flushing can
/// wait on C-owned job synchronization. Rust owns the bounded input copy, the
/// ordering of those operations, the end-directive adjustments, and the public
/// progress/return policy.
#[allow(clippy::too_many_arguments)]
unsafe fn compress_stream_generic_with<T, J, F, S>(
    context: ZSTDMT_compressStreamContextProjection,
    input: ZSTDMT_streamInputProjection,
    output: ZSTDMT_streamOutputProjection,
    end_op: c_uint,
    mut try_get_input_range: T,
    mut create_job: J,
    mut flush_produced: F,
    mut find_sync_point: S,
) -> ZSTDMT_compressStreamResult
where
    T: FnMut(&mut ZSTDMT_streamInputRangeProjection) -> c_int,
    J: FnMut(usize, c_uint) -> ZSTDMT_streamCreateJobResult,
    F: FnMut(*mut c_void, usize, usize, c_uint, c_uint) -> ZSTDMT_streamFlushResult,
    S: FnMut(
        &ZSTDMT_compressStreamContextProjection,
        &ZSTDMT_streamInputProjection,
        &ZSTDMT_streamInputRangeProjection,
    ) -> ZSTDMT_syncPointProjection,
{
    let mut input_pos = input.pos;
    let output_pos = output.pos;
    let initial_in_buff_filled =
        publishable_in_buff_filled(context.inBuffStart, context.inBuffFilled);
    if input_pos > input.size
        || output_pos > output.size
        || context.targetSectionSize == 0
        || context.inBuffFilled > context.inBuffCapacity
        || (context.inBuffStart.is_null() && context.inBuffFilled != 0)
    {
        return compress_stream_result(
            ERROR(ZstdErrorCode::Generic),
            input_pos,
            output_pos,
            initial_in_buff_filled,
        );
    }

    if context.frameEnded != 0 && end_op == ZSTD_E_CONTINUE {
        return compress_stream_result(
            ERROR(ZstdErrorCode::StageWrong),
            input_pos,
            output_pos,
            initial_in_buff_filled,
        );
    }

    let mut end_op = end_op;
    let mut input_range = ZSTDMT_streamInputRangeProjection {
        bufferStart: context.inBuffStart,
        bufferCapacity: context.inBuffCapacity,
        bufferFilled: context.inBuffFilled,
    };
    let mut forward_input_progress = false;

    /* Fill the current round-buffer section unless a prepared job is waiting
     * for a worker. This is the same ordering as the C implementation: a
     * blocked range acquisition is allowed to fall through to the flush pass.
     */
    if context.jobReady == 0 && input.size > input_pos {
        if input_range.bufferStart.is_null() {
            debug_assert_eq!(input_range.bufferFilled, 0);
            let _ = try_get_input_range(&mut input_range);
        }

        if !input_range.bufferStart.is_null() {
            let sync_point = find_sync_point(&context, &input, &input_range);
            if sync_point.flush != 0 && end_op == ZSTD_E_CONTINUE {
                end_op = ZSTD_E_FLUSH;
            }

            let available = input.size - input_pos;
            let Some(buffer_left) = input_range
                .bufferCapacity
                .checked_sub(input_range.bufferFilled)
            else {
                return compress_stream_result(
                    ERROR(ZstdErrorCode::Generic),
                    input_pos,
                    output_pos,
                    publishable_in_buff_filled(input_range.bufferStart, input_range.bufferFilled),
                );
            };
            if sync_point.toLoad > available
                || sync_point.toLoad > buffer_left
                || (sync_point.toLoad != 0 && input.src.is_null())
            {
                return compress_stream_result(
                    ERROR(ZstdErrorCode::Generic),
                    input_pos,
                    output_pos,
                    publishable_in_buff_filled(input_range.bufferStart, input_range.bufferFilled),
                );
            }

            if sync_point.toLoad != 0 {
                let source = input
                    .src
                    .cast::<u8>()
                    .wrapping_add(input_pos)
                    .cast::<c_void>();
                unsafe {
                    ptr::copy_nonoverlapping(
                        source.cast::<u8>(),
                        input_range
                            .bufferStart
                            .cast::<u8>()
                            .add(input_range.bufferFilled),
                        sync_point.toLoad,
                    );
                }
                input_pos += sync_point.toLoad;
                input_range.bufferFilled += sync_point.toLoad;
                forward_input_progress = true;
            }
        }
    }

    if input_pos < input.size && end_op == ZSTD_E_END {
        /* The caller cannot end a frame while input remains outside the
         * current job. The current pass must flush this job first. */
        end_op = ZSTD_E_FLUSH;
    }

    let mut returned_in_buff_filled =
        publishable_in_buff_filled(input_range.bufferStart, input_range.bufferFilled);
    if context.jobReady != 0
        || input_range.bufferFilled >= context.targetSectionSize
        || (end_op != ZSTD_E_CONTINUE && input_range.bufferFilled > 0)
        || (end_op == ZSTD_E_END && context.frameEnded == 0)
    {
        debug_assert!(input_range.bufferFilled <= context.targetSectionSize);
        let create_result = create_job(input_range.bufferFilled, end_op);
        returned_in_buff_filled = if create_result.inBuffActive != 0 {
            input_range.bufferFilled
        } else {
            0
        };
        if ERR_isError(create_result.result) {
            return compress_stream_result(
                create_result.result,
                input_pos,
                output_pos,
                returned_in_buff_filled,
            );
        }
    }

    let flush_result = flush_produced(
        output.dst,
        output.size,
        output_pos,
        u32::from(!forward_input_progress),
        end_op,
    );
    let remaining_to_flush = flush_result.result;
    let result = if input_pos < input.size {
        remaining_to_flush.max(1)
    } else {
        remaining_to_flush
    };
    compress_stream_result(
        result,
        input_pos,
        flush_result.outputPos,
        returned_in_buff_filled,
    )
}

/// C ABI entry point for the MT outer scheduling/flush policy.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_compressStreamGeneric(
    context: *const ZSTDMT_compressStreamContextProjection,
    input: *const ZSTDMT_streamInputProjection,
    output: *const ZSTDMT_streamOutputProjection,
    end_op: c_uint,
    opaque: *mut c_void,
    tryGetInputRange: Option<ZSTDMT_streamTryGetInputRangeFn>,
    createJob: Option<ZSTDMT_streamCreateJobFn>,
    flushProduced: Option<ZSTDMT_streamFlushProducedFn>,
) -> ZSTDMT_compressStreamResult {
    let Some(context) = (unsafe { context.as_ref() }).copied() else {
        return compress_stream_result(ERROR(ZstdErrorCode::Generic), 0, 0, 0);
    };
    let Some(input) = (unsafe { input.as_ref() }).copied() else {
        return compress_stream_result(ERROR(ZstdErrorCode::Generic), 0, 0, context.inBuffFilled);
    };
    let Some(output) = (unsafe { output.as_ref() }).copied() else {
        return compress_stream_result(
            ERROR(ZstdErrorCode::Generic),
            input.pos,
            0,
            context.inBuffFilled,
        );
    };
    let (Some(try_get_input_range), Some(create_job), Some(flush_produced)) =
        (tryGetInputRange, createJob, flushProduced)
    else {
        return compress_stream_result(
            ERROR(ZstdErrorCode::Generic),
            input.pos,
            output.pos,
            context.inBuffFilled,
        );
    };

    unsafe {
        compress_stream_generic_with(
            context,
            input,
            output,
            end_op,
            |projection| try_get_input_range(opaque, projection),
            |src_size, end| create_job(opaque, src_size, end),
            |dst, size, pos, block_to_flush, end| {
                flush_produced(opaque, dst, size, pos, block_to_flush, end)
            },
            |context, input, input_range| {
                let (to_load, flush) = find_synchronization_point(
                    input.src,
                    input.size,
                    input.pos,
                    context.targetSectionSize,
                    input_range.bufferStart.cast_const(),
                    input_range.bufferFilled,
                    context.rsyncable,
                    context.rsyncPrimePower,
                    context.rsyncHitMask,
                );
                ZSTDMT_syncPointProjection {
                    toLoad: to_load,
                    flush,
                }
            },
        )
    }
}

#[inline]
fn get_input_data_in_use_with<F>(
    first_job_id: c_uint,
    last_job_id: c_uint,
    job_id_mask: c_uint,
    round_buffer_capacity: usize,
    target_section_size: usize,
    mut project_job: F,
) -> ZSTDMT_inputRange
where
    F: FnMut(c_uint) -> ZSTDMT_jobProjection,
{
    debug_assert!(target_section_size > 0);
    if target_section_size == 0 {
        return ZSTDMT_inputRange::default();
    }

    /* No job can refer to the first round buffer before this point. */
    let jobs_before_round_buffer_reuse = round_buffer_capacity / target_section_size;
    /* C's usual arithmetic conversions promote `lastJobID` to size_t here. */
    if (last_job_id as usize) < jobs_before_round_buffer_reuse {
        return ZSTDMT_inputRange::default();
    }

    for job_id in first_job_id..last_job_id {
        let projection = project_job(job_id & job_id_mask);
        if projection.consumed >= projection.srcSize {
            continue;
        }

        if projection.prefixSize > 0 {
            debug_assert!((projection.prefixStart as usize) <= (projection.srcStart as usize));
            return ZSTDMT_inputRange {
                start: projection.prefixStart,
                size: projection.prefixSize,
            };
        }
        return ZSTDMT_inputRange {
            start: projection.srcStart,
            size: projection.srcSize,
        };
    }

    ZSTDMT_inputRange::default()
}

/// Find the earliest still-in-use input range in the C-owned job ring.
///
/// C supplies each snapshot under its private job mutex. Rust performs the
/// ring ordering and prefix/source selection without receiving the job table
/// or any of its private synchronization layout.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_getInputDataInUse(
    firstJobID: c_uint,
    lastJobID: c_uint,
    jobIDMask: c_uint,
    roundBufferCapacity: usize,
    targetSectionSize: usize,
    opaque: *mut c_void,
    projectJob: Option<ZSTDMT_jobProjectionFn>,
) -> ZSTDMT_inputRange {
    let Some(project_job) = projectJob else {
        return ZSTDMT_inputRange::default();
    };
    get_input_data_in_use_with(
        firstJobID,
        lastJobID,
        jobIDMask,
        roundBufferCapacity,
        targetSectionSize,
        |job_id| {
            let mut projection = ZSTDMT_jobProjection::default();
            unsafe { project_job(opaque, job_id, &mut projection) };
            projection
        },
    )
}

#[inline]
fn to_flush_now_with<F>(
    done_job_id: c_uint,
    next_job_id: c_uint,
    job_id_mask: c_uint,
    mut project_job: F,
) -> usize
where
    F: FnMut(c_uint) -> ZSTDMT_jobProjection,
{
    if done_job_id == next_job_id {
        return 0;
    }

    let projection = project_job(done_job_id & job_id_mask);
    let produced = if ERR_isError(projection.cSize) {
        0
    } else {
        projection.cSize
    };
    let flushed = if ERR_isError(projection.cSize) {
        0
    } else {
        projection.dstFlushed
    };
    debug_assert!(flushed <= produced);
    debug_assert!(projection.consumed <= projection.srcSize);

    let to_flush = produced.wrapping_sub(flushed);
    if to_flush == 0 {
        /* A live job with no output yet must still have input remaining. */
        debug_assert!(projection.consumed < projection.srcSize);
    }
    to_flush
}

/// Return the output already produced by the oldest active C job but not yet
/// published to the caller's output buffer.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_toFlushNow(
    doneJobID: c_uint,
    nextJobID: c_uint,
    jobIDMask: c_uint,
    opaque: *mut c_void,
    projectJob: Option<ZSTDMT_jobProjectionFn>,
) -> usize {
    let Some(project_job) = projectJob else {
        return 0;
    };
    to_flush_now_with(doneJobID, nextJobID, jobIDMask, |job_id| {
        let mut projection = ZSTDMT_jobProjection::default();
        unsafe { project_job(opaque, job_id, &mut projection) };
        projection
    })
}

#[inline]
fn try_get_input_range_with(
    projection: ZSTDMT_tryGetInputRangeProjection,
) -> ZSTDMT_tryGetInputRangeResult {
    let invalid = || ZSTDMT_tryGetInputRangeResult::default();

    debug_assert!(projection.roundBufferPos <= projection.roundBufferCapacity);
    debug_assert!(projection.targetSectionSize > 0);
    if projection.roundBufferPos > projection.roundBufferCapacity
        || projection.targetSectionSize == 0
        || projection.targetSectionSize > projection.roundBufferCapacity
    {
        return invalid();
    }

    let space_left = projection.roundBufferCapacity - projection.roundBufferPos;
    let (round_buffer_pos, move_prefix) = if space_left < projection.targetSectionSize {
        /* The C implementation first checks the prefix range at the start of
         * the round buffer before resetting the position there. */
        if is_overlapped(
            projection.roundBufferStart,
            projection.prefixSize,
            projection.inUseStart,
            projection.inUseSize,
        ) != 0
        {
            return invalid();
        }
        if projection.prefixSize > projection.roundBufferCapacity - projection.targetSectionSize {
            return invalid();
        }
        (projection.prefixSize, 1)
    } else {
        (projection.roundBufferPos, 0)
    };

    let buffer_start = projection
        .roundBufferStart
        .cast::<u8>()
        .wrapping_add(round_buffer_pos)
        .cast_mut()
        .cast::<c_void>();
    if move_prefix == 0 {
        /* This is an internal invariant in the original C helper: a live
         * prefix must never be overwritten by the next source section. */
        debug_assert_eq!(
            is_overlapped(
                buffer_start,
                projection.targetSectionSize,
                projection.prefixStart,
                projection.prefixSize,
            ),
            0
        );
    }
    if is_overlapped(
        buffer_start,
        projection.targetSectionSize,
        projection.inUseStart,
        projection.inUseSize,
    ) != 0
    {
        return invalid();
    }

    ZSTDMT_tryGetInputRangeResult {
        ready: 1,
        movePrefix: move_prefix,
        bufferStart: buffer_start,
        bufferCapacity: projection.targetSectionSize,
        roundBufferPos: round_buffer_pos,
    }
}

/// Select the next reusable round-buffer section for MT input.  The C adapter
/// retains all synchronization, prefix copying, and private context updates.
#[cfg(not(test))]
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_tryGetInputRange(
    projection: *const ZSTDMT_tryGetInputRangeProjection,
) -> ZSTDMT_tryGetInputRangeResult {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return ZSTDMT_tryGetInputRangeResult::default();
    };
    try_get_input_range_with(projection)
}

#[inline]
fn cycle_log(chain_log: c_uint, strategy: c_int) -> c_uint {
    chain_log.wrapping_sub((strategy >= ZSTD_BTLAZY2) as c_uint)
}

#[inline]
fn compute_target_job_log(
    window_log: c_uint,
    chain_log: c_uint,
    strategy: c_int,
    enable_ldm: c_int,
) -> c_uint {
    let job_log = if enable_ldm == ZSTD_PS_ENABLE {
        /* In Long Range Mode, the windowLog is typically oversized.
         * In which case, it's preferable to determine the jobSize
         * based on cycleLog instead. */
        21.max(cycle_log(chain_log, strategy).wrapping_add(3))
    } else {
        20.max(window_log.wrapping_add(2))
    };
    job_log.min(ZSTDMT_JOBLOG_MAX)
}

#[inline]
fn overlap_log_default(strategy: c_int) -> c_int {
    match strategy {
        ZSTD_BTULTRA2 => 9,
        ZSTD_BTULTRA | ZSTD_BTOPT => 8,
        ZSTD_BTLAZY2 | ZSTD_LAZY2 => 7,
        ZSTD_LAZY | ZSTD_GREEDY | ZSTD_DFAST | ZSTD_FAST => 6,
        _ => 6,
    }
}

#[inline]
fn overlap_log(overlap_log: c_int, strategy: c_int) -> c_int {
    debug_assert!((0..=9).contains(&overlap_log));
    if overlap_log == 0 {
        overlap_log_default(strategy)
    } else {
        overlap_log
    }
}

#[inline]
fn compute_overlap_size(
    window_log: c_uint,
    chain_log: c_uint,
    strategy: c_int,
    overlap_log_value: c_int,
    enable_ldm: c_int,
) -> usize {
    let overlap_r_log = 9 - overlap_log(overlap_log_value, strategy);
    let mut overlap_log_value = if overlap_r_log >= 8 {
        0
    } else {
        window_log as c_int - overlap_r_log
    };
    debug_assert!((0..=8).contains(&overlap_r_log));
    if enable_ldm == ZSTD_PS_ENABLE {
        /* In Long Range Mode, the windowLog is typically oversized.
         * In which case, it's preferable to determine the jobSize
         * based on chainLog instead.
         * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */
        let target_job_log = compute_target_job_log(window_log, chain_log, strategy, enable_ldm);
        overlap_log_value = window_log.min(target_job_log.wrapping_sub(2)) as c_int - overlap_r_log;
    }
    debug_assert!(overlap_log_value >= 0);
    debug_assert!(overlap_log_value <= ZSTD_WINDOWLOG_MAX as c_int);
    if overlap_log_value == 0 {
        0
    } else {
        1usize << overlap_log_value as usize
    }
}

/// C ABI for the pure MT target job-log policy.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_computeTargetJobLog(
    windowLog: c_uint,
    chainLog: c_uint,
    strategy: c_int,
    enableLdm: c_int,
) -> c_uint {
    compute_target_job_log(windowLog, chainLog, strategy, enableLdm)
}

/// C ABI for the MT overlap-log default/selection policy.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_overlapLog(overlapLog: c_int, strategy: c_int) -> c_int {
    overlap_log(overlapLog, strategy)
}

/// C ABI for the pure MT overlap-size policy.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_computeOverlapSize(
    windowLog: c_uint,
    chainLog: c_uint,
    strategy: c_int,
    overlapLog: c_int,
    enableLdm: c_int,
) -> usize {
    compute_overlap_size(windowLog, chainLog, strategy, overlapLog, enableLdm)
}

#[inline]
fn frame_progression(
    consumed: u64,
    in_buff_filled: usize,
    produced: u64,
    current_job_id: c_uint,
) -> ZSTD_frameProgression {
    ZSTD_frameProgression {
        // C's usual arithmetic conversions promote size_t to U64 before wrapping.
        ingested: consumed.wrapping_add(in_buff_filled as u64),
        consumed,
        produced,
        flushed: produced,
        currentJobID: current_job_id,
        nbActiveWorkers: 0,
    }
}

#[inline]
fn frame_progression_add_job(
    mut progression: ZSTD_frameProgression,
    src_size: usize,
    consumed: usize,
    produced: usize,
    flushed: usize,
) -> ZSTD_frameProgression {
    progression.ingested = progression.ingested.wrapping_add(src_size as u64);
    progression.consumed = progression.consumed.wrapping_add(consumed as u64);
    progression.produced = progression.produced.wrapping_add(produced as u64);
    progression.flushed = progression.flushed.wrapping_add(flushed as u64);
    progression.nbActiveWorkers = progression
        .nbActiveWorkers
        .wrapping_add((consumed < src_size) as c_uint);
    progression
}

#[inline]
fn frame_progression_with_jobs<F>(
    consumed: u64,
    in_buff_filled: usize,
    produced: u64,
    current_job_id: c_uint,
    done_job_id: c_uint,
    next_job_id: c_uint,
    job_ready: c_uint,
    job_id_mask: c_uint,
    mut project_job: F,
) -> ZSTD_frameProgression
where
    F: FnMut(c_uint) -> ZSTDMT_jobProjection,
{
    let mut progression = frame_progression(consumed, in_buff_filled, produced, current_job_id);
    debug_assert!(job_ready <= 1);
    let last_job_id = next_job_id.wrapping_add(job_ready);
    for job_id in done_job_id..last_job_id {
        let projection = project_job(job_id & job_id_mask);
        let (produced, flushed) = if ERR_isError(projection.cSize) {
            (0, 0)
        } else {
            (projection.cSize, projection.dstFlushed)
        };
        debug_assert!(flushed <= produced);
        progression = frame_progression_add_job(
            progression,
            projection.srcSize,
            projection.consumed,
            produced,
            flushed,
        );
    }
    progression
}

/// Construct the base MT frame progression from C-owned scalar state.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_frameProgression(
    consumed: u64,
    in_buff_filled: usize,
    produced: u64,
    current_job_id: c_uint,
) -> ZSTD_frameProgression {
    frame_progression(consumed, in_buff_filled, produced, current_job_id)
}

/// Add one C-normalized, mutex-protected job snapshot to MT frame progression.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_frameProgressionAddJob(
    progression: ZSTD_frameProgression,
    src_size: usize,
    consumed: usize,
    produced: usize,
    flushed: usize,
) -> ZSTD_frameProgression {
    frame_progression_add_job(progression, src_size, consumed, produced, flushed)
}

/// Aggregate the base MT progression with mutex-protected C job snapshots.
/// Rust owns only the ring scan and normalization policy; C retains the job
/// descriptor layout and takes each descriptor mutex in `projectJob`.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_frameProgressionWithJobs(
    consumed: u64,
    in_buff_filled: usize,
    produced: u64,
    current_job_id: c_uint,
    done_job_id: c_uint,
    next_job_id: c_uint,
    job_ready: c_uint,
    job_id_mask: c_uint,
    opaque: *mut c_void,
    project_job: Option<ZSTDMT_jobProjectionFn>,
) -> ZSTD_frameProgression {
    let Some(project_job) = project_job else {
        return frame_progression(consumed, in_buff_filled, produced, current_job_id);
    };
    frame_progression_with_jobs(
        consumed,
        in_buff_filled,
        produced,
        current_job_id,
        done_job_id,
        next_job_id,
        job_ready,
        job_id_mask,
        |job_id| {
            let mut projection = ZSTDMT_jobProjection::default();
            unsafe { project_job(opaque, job_id, &mut projection) };
            projection
        },
    )
}

#[inline]
unsafe fn rolling_hash_append(mut hash: u64, input: *const u8, size: usize) -> u64 {
    for pos in 0..size {
        let byte = u64::from(unsafe { *input.wrapping_add(pos) });
        hash = hash
            .wrapping_mul(PRIME8_BYTES)
            .wrapping_add(byte.wrapping_add(ROLL_HASH_CHAR_OFFSET));
    }
    hash
}

#[inline]
unsafe fn rolling_hash_compute(input: *const u8, size: usize) -> u64 {
    unsafe { rolling_hash_append(0, input, size) }
}

#[inline]
fn rolling_hash_rotate(hash: u64, to_remove: u8, to_add: u8, prime_power: u64) -> u64 {
    hash.wrapping_sub(
        u64::from(to_remove)
            .wrapping_add(ROLL_HASH_CHAR_OFFSET)
            .wrapping_mul(prime_power),
    )
    .wrapping_mul(PRIME8_BYTES)
    .wrapping_add(u64::from(to_add).wrapping_add(ROLL_HASH_CHAR_OFFSET))
}

#[inline]
fn rolling_hash_prime_power(length: c_uint) -> u64 {
    let mut base = PRIME8_BYTES;
    let mut exponent = u64::from(length.wrapping_sub(1));
    let mut power: u64 = 1;
    while exponent != 0 {
        if exponent & 1 != 0 {
            power = power.wrapping_mul(base);
        }
        exponent >>= 1;
        base = base.wrapping_mul(base);
    }
    power
}

/// C ABI for the rolling-hash prime power used by rsyncable MT compression.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_rollingHashPrimePower(length: c_uint) -> u64 {
    rolling_hash_prime_power(length)
}

/// Search for a synchronization point without exposing the C MT context or
/// its private buffer types across the FFI boundary.
#[inline]
unsafe fn find_synchronization_point(
    input_src: *const c_void,
    input_size: usize,
    input_pos: usize,
    target_section_size: usize,
    in_buff_start: *const c_void,
    in_buff_filled: usize,
    rsyncable: c_int,
    prime_power: u64,
    hit_mask: u64,
) -> (usize, c_int) {
    let istart = input_src.cast::<u8>().wrapping_add(input_pos);
    let available = input_size.wrapping_sub(input_pos);
    let mut to_load = available.min(target_section_size.wrapping_sub(in_buff_filled));
    let mut flush = 0;

    if rsyncable == 0 {
        return (to_load, flush);
    }
    if in_buff_filled.wrapping_add(available) < RSYNC_MIN_BLOCK_SIZE {
        return (to_load, flush);
    }
    if in_buff_filled.wrapping_add(to_load) < RSYNC_LENGTH {
        return (to_load, flush);
    }

    let (mut pos, prev, mut hash) = if in_buff_filled < RSYNC_MIN_BLOCK_SIZE {
        let pos = RSYNC_MIN_BLOCK_SIZE.wrapping_sub(in_buff_filled);
        if pos >= RSYNC_LENGTH {
            let prev = istart.wrapping_add(pos.wrapping_sub(RSYNC_LENGTH));
            let hash = unsafe { rolling_hash_compute(prev, RSYNC_LENGTH) };
            (pos, prev, hash)
        } else {
            debug_assert!(in_buff_filled >= RSYNC_LENGTH);
            let prev = in_buff_start
                .cast::<u8>()
                .wrapping_add(in_buff_filled.wrapping_sub(RSYNC_LENGTH));
            let hash = unsafe { rolling_hash_compute(prev.wrapping_add(pos), RSYNC_LENGTH - pos) };
            let hash = unsafe { rolling_hash_append(hash, istart, pos) };
            (pos, prev, hash)
        }
    } else {
        let pos = 0;
        let prev = in_buff_start
            .cast::<u8>()
            .wrapping_add(in_buff_filled.wrapping_sub(RSYNC_LENGTH));
        let hash = unsafe { rolling_hash_compute(prev, RSYNC_LENGTH) };
        if (hash & hit_mask) == hit_mask {
            return (0, 1);
        }
        (pos, prev, hash)
    };

    debug_assert!(
        pos < RSYNC_LENGTH
            || unsafe {
                rolling_hash_compute(
                    istart.wrapping_add(pos.wrapping_sub(RSYNC_LENGTH)),
                    RSYNC_LENGTH,
                ) == hash
            }
    );
    while pos < to_load {
        let to_remove = if pos < RSYNC_LENGTH {
            unsafe { *prev.wrapping_add(pos) }
        } else {
            unsafe { *istart.wrapping_add(pos.wrapping_sub(RSYNC_LENGTH)) }
        };
        let to_add = unsafe { *istart.wrapping_add(pos) };
        hash = rolling_hash_rotate(hash, to_remove, to_add, prime_power);
        debug_assert!(in_buff_filled.wrapping_add(pos) >= RSYNC_MIN_BLOCK_SIZE);
        if (hash & hit_mask) == hit_mask {
            to_load = pos + 1;
            flush = 1;
            pos += 1;
            break;
        }
        pos += 1;
    }
    debug_assert!(
        pos < RSYNC_LENGTH
            || unsafe {
                rolling_hash_compute(
                    istart.wrapping_add(pos.wrapping_sub(RSYNC_LENGTH)),
                    RSYNC_LENGTH,
                ) == hash
            }
    );
    (to_load, flush)
}

/// C ABI for the pure MT synchronization-point policy.
///
/// The caller retains ownership of the MT context, input position, and
/// buffer updates.  Rust only scans the two byte ranges and writes the two
/// scalar results.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_findSynchronizationPoint(
    inputSrc: *const c_void,
    inputSize: usize,
    inputPos: usize,
    targetSectionSize: usize,
    inBuffStart: *const c_void,
    inBuffFilled: usize,
    rsyncable: c_int,
    primePower: u64,
    hitMask: u64,
    toLoad: *mut usize,
    flush: *mut c_int,
) {
    let (computed_to_load, computed_flush) = unsafe {
        find_synchronization_point(
            inputSrc,
            inputSize,
            inputPos,
            targetSectionSize,
            inBuffStart,
            inBuffFilled,
            rsyncable,
            primePower,
            hitMask,
        )
    };
    unsafe {
        *toLoad = computed_to_load;
        *flush = computed_flush;
    }
}

type ZstdAllocFunction = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
type ZstdFreeFunction = unsafe extern "C" fn(*mut c_void, *mut c_void);

/// ABI-compatible representation of `ZSTD_customMem` from `zstd.h`.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZstdCustomMem {
    pub customAlloc: Option<ZstdAllocFunction>,
    pub customFree: Option<ZstdFreeFunction>,
    pub opaque: *mut c_void,
}

/// ABI-compatible buffer returned to the C adapter.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZstdMtBuffer {
    pub start: *mut c_void,
    pub capacity: usize,
}

/// ABI-compatible representation of the private C `ZSTD_window_t` used by
/// the MT LDM overlap callback.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZstdMtWindow {
    pub nextSrc: *const u8,
    pub base: *const u8,
    pub dictBase: *const u8,
    pub dictLimit: u32,
    pub lowLimit: u32,
    pub nbOverflowCorrections: u32,
}

const _: () = {
    assert!(std::mem::offset_of!(ZstdMtWindow, nextSrc) == 0);
    assert!(std::mem::offset_of!(ZstdMtWindow, base) == size_of::<usize>());
    assert!(std::mem::offset_of!(ZstdMtWindow, dictBase) == 2 * size_of::<usize>());
    assert!(std::mem::offset_of!(ZstdMtWindow, dictLimit) == 3 * size_of::<usize>());
    assert!(size_of::<ZstdMtWindow>() == if size_of::<usize>() == 8 { 40 } else { 24 });
};

/// ABI-compatible representation of `rawSeq` from `zstd_compress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZstdMtRawSeq {
    pub offset: u32,
    pub litLength: u32,
    pub matchLength: u32,
}

/// ABI-compatible representation of `RawSeqStore_t` from
/// `zstd_compress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct ZstdMtRawSeqStore {
    pub seq: *mut ZstdMtRawSeq,
    pub pos: usize,
    pub posInSequence: usize,
    pub size: usize,
    pub capacity: usize,
}

#[inline]
fn buffer_to_seq(buffer: ZstdMtBuffer) -> ZstdMtRawSeqStore {
    ZstdMtRawSeqStore {
        seq: buffer.start.cast::<ZstdMtRawSeq>(),
        pos: 0,
        posInSequence: 0,
        size: 0,
        capacity: buffer.capacity / mem::size_of::<ZstdMtRawSeq>(),
    }
}

#[inline]
fn seq_to_buffer(seq: ZstdMtRawSeqStore) -> ZstdMtBuffer {
    ZstdMtBuffer {
        start: seq.seq.cast::<c_void>(),
        capacity: seq.capacity.wrapping_mul(mem::size_of::<ZstdMtRawSeq>()),
    }
}

#[inline]
fn is_overlapped(
    buffer_start: *const c_void,
    buffer_capacity: usize,
    range_start: *const c_void,
    range_size: usize,
) -> c_int {
    if buffer_start.is_null() || range_start.is_null() {
        return 0;
    }

    let buffer_start = buffer_start.cast::<u8>();
    let range_start = range_start.cast::<u8>();
    let buffer_end = buffer_start.wrapping_add(buffer_capacity);
    let range_end = range_start.wrapping_add(range_size);

    /* Empty ranges cannot overlap. */
    if buffer_start == buffer_end || range_start == range_end {
        return 0;
    }

    (buffer_start < range_end && range_start < buffer_end) as c_int
}

/// Convert a byte buffer into the raw-sequence store view used by the MT
/// sequence pool. The capacity is expressed in whole `rawSeq` elements, just
/// like the original C conversion leaf.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_bufferToSeq(buffer: ZstdMtBuffer) -> ZstdMtRawSeqStore {
    buffer_to_seq(buffer)
}

/// Convert the raw-sequence store view back to a byte buffer for pool APIs.
/// The returned capacity is measured in bytes and follows C `size_t` wraparound
/// semantics for the multiplication.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_seqToBuffer(seq: ZstdMtRawSeqStore) -> ZstdMtBuffer {
    seq_to_buffer(seq)
}

/// Return non-zero when two non-empty byte ranges overlap.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_isOverlapped(
    bufferStart: *const c_void,
    bufferCapacity: usize,
    rangeStart: *const c_void,
    rangeSize: usize,
) -> c_int {
    is_overlapped(bufferStart, bufferCapacity, rangeStart, rangeSize)
}

/// Return non-zero when a buffer overlaps either the external dictionary or
/// the active prefix represented by a C `ZSTD_window_t`.
///
/// The C window stores the two ranges as pointer/index pairs.  Passing those
/// scalar fields separately keeps the private C struct out of the Rust ABI;
/// the byte-distance arithmetic mirrors the original pointer subtraction.
#[no_mangle]
pub extern "C" fn ZSTDMT_rust_doesOverlapWindow(
    bufferStart: *const c_void,
    bufferCapacity: usize,
    nextSrc: *const c_void,
    base: *const c_void,
    dictBase: *const c_void,
    dictLimit: u32,
    lowLimit: u32,
) -> c_int {
    let ext_dict_start = dictBase.cast::<u8>().wrapping_add(lowLimit as usize);
    let ext_dict_size = dictLimit.wrapping_sub(lowLimit) as usize;
    let prefix_start = base.cast::<u8>().wrapping_add(dictLimit as usize);
    let prefix_size = (nextSrc as usize)
        .wrapping_sub(base as usize)
        .wrapping_sub(dictLimit as usize);

    (is_overlapped(
        bufferStart,
        bufferCapacity,
        ext_dict_start.cast(),
        ext_dict_size,
    ) != 0
        || is_overlapped(
            bufferStart,
            bufferCapacity,
            prefix_start.cast(),
            prefix_size,
        ) != 0) as c_int
}

/// Return non-zero when a buffer overlaps either range in a C window.
///
/// This keeps the original by-value internal ABI while the scalar helper
/// above remains available to projections that cannot expose the private
/// window layout.
#[no_mangle]
pub extern "C" fn ZSTDMT_doesOverlapWindow(buffer: ZstdMtBuffer, window: ZstdMtWindow) -> c_int {
    ZSTDMT_rust_doesOverlapWindow(
        buffer.start as *const c_void,
        buffer.capacity,
        window.nextSrc.cast(),
        window.base.cast(),
        window.dictBase.cast(),
        window.dictLimit,
        window.lowLimit,
    )
}

#[derive(Default)]
struct BufferPoolState {
    buffer_size: usize,
    nb_buffers: usize,
}

/// The object is allocated with the caller's `ZSTD_customMem`; its mutex
/// protects all mutable pool state and the raw reusable-buffer array.
pub struct RustBufferPool {
    custom_mem: ZstdCustomMem,
    total_buffers: usize,
    buffers: *mut MaybeUninit<ZstdMtBuffer>,
    state: Mutex<BufferPoolState>,
}

// The raw array is only accessed while `state` is held.  The pool itself is
// passed between C worker threads as an opaque pointer.
unsafe impl Send for RustBufferPool {}
unsafe impl Sync for RustBufferPool {}

/// The `ZSTD_CCtx *` values are created and destroyed by the existing C API;
/// Rust owns only the synchronized reusable-pointer pool.
pub struct RustCCtxPool {
    custom_mem: ZstdCustomMem,
    total_cctx: usize,
    cctxs: *mut MaybeUninit<*mut c_void>,
    state: Mutex<usize>,
}

unsafe impl Send for RustCCtxPool {}
unsafe impl Sync for RustCCtxPool {}

unsafe extern "C" {
    fn ZSTD_createCCtx_advanced(custom_mem: ZstdCustomMem) -> *mut c_void;
    fn ZSTD_freeCCtx(cctx: *mut c_void) -> usize;
    fn ZSTD_sizeof_CCtx(cctx: *const c_void) -> usize;
}

unsafe fn custom_calloc(size: usize, custom_mem: ZstdCustomMem) -> *mut c_void {
    if let Some(alloc) = custom_mem.customAlloc {
        let allocation = unsafe { alloc(custom_mem.opaque, size) };
        if !allocation.is_null() {
            unsafe { ptr::write_bytes(allocation, 0, size) };
        }
        allocation
    } else {
        unsafe { libc::calloc(1, size) }
    }
}

unsafe fn custom_malloc(size: usize, custom_mem: ZstdCustomMem) -> *mut c_void {
    if let Some(alloc) = custom_mem.customAlloc {
        unsafe { alloc(custom_mem.opaque, size) }
    } else {
        unsafe { libc::malloc(size) }
    }
}

unsafe fn custom_free(allocation: *mut c_void, custom_mem: ZstdCustomMem) {
    if allocation.is_null() {
        return;
    }
    if let Some(free) = custom_mem.customFree {
        unsafe { free(custom_mem.opaque, allocation) };
    } else {
        unsafe { libc::free(allocation) };
    }
}

fn checked_array_size<T>(len: usize) -> Option<usize> {
    mem::size_of::<T>().checked_mul(len)
}

fn rounded_job_count(requested: c_uint) -> Option<c_uint> {
    if requested == 0 {
        return None;
    }
    // The original C expression is `1 << (highbit(requested) + 1)`, which
    // deliberately chooses a strictly larger power of two when requested is
    // already a power of two.
    let shift = usize::BITS - (requested as usize).leading_zeros();
    let count = 1usize.checked_shl(shift)?;
    if count > c_uint::MAX as usize {
        return None;
    }
    Some(count as c_uint)
}

unsafe fn create_job_table(
    nb_jobs_ptr: *mut c_uint,
    job_size: usize,
    custom_mem: ZstdCustomMem,
) -> *mut c_void {
    if nb_jobs_ptr.is_null() || job_size == 0 {
        return ptr::null_mut();
    }
    let Some(nb_jobs) = (unsafe { rounded_job_count(*nb_jobs_ptr) }) else {
        return ptr::null_mut();
    };
    let Some(table_size) = job_size.checked_mul(nb_jobs as usize) else {
        return ptr::null_mut();
    };
    let table = unsafe { custom_calloc(table_size, custom_mem) };
    if table.is_null() {
        return ptr::null_mut();
    }

    unsafe { *nb_jobs_ptr = nb_jobs };
    table
}

unsafe fn free_job_table_storage(job_table: *mut c_void, custom_mem: ZstdCustomMem) {
    if job_table.is_null() {
        return;
    }
    unsafe { custom_free(job_table, custom_mem) };
}

type JobTableInitSync = unsafe extern "C" fn(*mut c_void, c_uint, usize) -> c_int;
type JobTableDestroySync = unsafe extern "C" fn(*mut c_void, c_uint, usize);

/// Projection for the final MT job-table release. Rust owns the
/// destroy-before-storage-free order; C retains the descriptor synchronization
/// callback and custom allocator behind this opaque projection.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTDMT_RustFreeJobsTableProjection {
    jobTable: *mut c_void,
    nbJobs: c_uint,
    jobSize: usize,
    customMem: ZstdCustomMem,
    destroySync: Option<JobTableDestroySync>,
}

const _: () = {
    assert!(size_of::<JobTableDestroySync>() == size_of::<usize>());
    assert!(offset_of!(ZSTDMT_RustFreeJobsTableProjection, jobTable) == 0);
    assert!(offset_of!(ZSTDMT_RustFreeJobsTableProjection, nbJobs) == size_of::<usize>());
    assert!(
        offset_of!(ZSTDMT_RustFreeJobsTableProjection, jobSize)
            == if size_of::<usize>() == 8 { 16 } else { 8 }
    );
    assert!(
        offset_of!(ZSTDMT_RustFreeJobsTableProjection, customMem)
            == offset_of!(ZSTDMT_RustFreeJobsTableProjection, jobSize) + size_of::<usize>()
    );
    assert!(
        offset_of!(ZSTDMT_RustFreeJobsTableProjection, destroySync)
            == offset_of!(ZSTDMT_RustFreeJobsTableProjection, customMem)
                + size_of::<ZstdCustomMem>()
    );
    assert!(
        size_of::<ZSTDMT_RustFreeJobsTableProjection>()
            == offset_of!(ZSTDMT_RustFreeJobsTableProjection, destroySync)
                + size_of::<JobTableDestroySync>()
    );
};

unsafe fn free_job_table_with_sync(
    job_table: *mut c_void,
    nb_jobs: c_uint,
    job_size: usize,
    custom_mem: ZstdCustomMem,
    destroy_sync: JobTableDestroySync,
) {
    let projection = ZSTDMT_RustFreeJobsTableProjection {
        jobTable: job_table,
        nbJobs: nb_jobs,
        jobSize: job_size,
        customMem: custom_mem,
        destroySync: Some(destroy_sync),
    };
    unsafe { free_job_table_with_sync_projection(projection) };
}

unsafe fn free_job_table_with_sync_projection(projection: ZSTDMT_RustFreeJobsTableProjection) {
    if projection.jobTable.is_null() {
        return;
    }
    let Some(destroy_sync) = projection.destroySync else {
        return;
    };
    unsafe {
        destroy_sync(projection.jobTable, projection.nbJobs, projection.jobSize);
        free_job_table_storage(projection.jobTable, projection.customMem);
    }
}

unsafe fn create_job_table_with_sync(
    nb_jobs_ptr: *mut c_uint,
    job_size: usize,
    custom_mem: ZstdCustomMem,
    init_sync: JobTableInitSync,
    destroy_sync: JobTableDestroySync,
) -> *mut c_void {
    let job_table = unsafe { create_job_table(nb_jobs_ptr, job_size, custom_mem) };
    if job_table.is_null() {
        return ptr::null_mut();
    }

    let nb_jobs = unsafe { *nb_jobs_ptr };
    if unsafe { init_sync(job_table, nb_jobs, job_size) } != 0 {
        unsafe { free_job_table_with_sync(job_table, nb_jobs, job_size, custom_mem, destroy_sync) };
        return ptr::null_mut();
    }

    job_table
}

unsafe fn expand_job_table(
    job_table_ptr: *mut *mut c_void,
    job_id_mask_ptr: *mut c_uint,
    nb_workers: c_uint,
    job_size: usize,
    custom_mem: ZstdCustomMem,
    init_sync: JobTableInitSync,
    destroy_sync: JobTableDestroySync,
) -> usize {
    if job_table_ptr.is_null() || job_id_mask_ptr.is_null() {
        return ERROR(ZstdErrorCode::Generic);
    }

    let requested_jobs = nb_workers.wrapping_add(2);
    let current_capacity = unsafe { (*job_id_mask_ptr).wrapping_add(1) };
    if requested_jobs <= current_capacity {
        return 0;
    }

    let old_job_table = unsafe { *job_table_ptr };
    unsafe {
        free_job_table_with_sync(
            old_job_table,
            current_capacity,
            job_size,
            custom_mem,
            destroy_sync,
        );
        // Match ZSTDMT_expandJobsTable(): after the old table is released, a
        // failed replacement leaves the C context with no table and a zero
        // mask. The C wrapper copies these scalar results back to its private
        // context only after this adapter returns.
        *job_table_ptr = ptr::null_mut();
        *job_id_mask_ptr = 0;
    }

    let mut nb_jobs = requested_jobs;
    let new_job_table = unsafe {
        create_job_table_with_sync(&mut nb_jobs, job_size, custom_mem, init_sync, destroy_sync)
    };
    if new_job_table.is_null() {
        return ERROR(ZstdErrorCode::MemoryAllocation);
    }

    debug_assert!(nb_jobs.is_power_of_two());
    unsafe {
        *job_table_ptr = new_job_table;
        *job_id_mask_ptr = nb_jobs.wrapping_sub(1);
    }
    0
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_job_table_create(
    nb_jobs_ptr: *mut c_uint,
    job_size: usize,
    custom_mem: ZstdCustomMem,
) -> *mut c_void {
    unsafe { create_job_table(nb_jobs_ptr, job_size, custom_mem) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_createJobsTable(
    nb_jobs_ptr: *mut c_uint,
    job_size: usize,
    custom_mem: ZstdCustomMem,
    init_sync: JobTableInitSync,
    destroy_sync: JobTableDestroySync,
) -> *mut c_void {
    unsafe {
        create_job_table_with_sync(nb_jobs_ptr, job_size, custom_mem, init_sync, destroy_sync)
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_job_table_free(
    job_table: *mut c_void,
    _nb_jobs: c_uint,
    _job_size: usize,
    custom_mem: ZstdCustomMem,
) {
    // The MT C adapter destroys its platform mutexes and condition variables
    // before calling this storage-only release function.  Keeping this Rust
    // side free of MT-only C references also preserves single-threaded builds
    // where zstdmt_compress.c is intentionally omitted.
    unsafe { free_job_table_storage(job_table, custom_mem) }
}

/// Release the C-owned synchronization objects before Rust releases the
/// opaque job-table storage. This is the normal teardown counterpart to the
/// same order used by failed create/expand transactions.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_freeJobsTable(
    projection: *const ZSTDMT_RustFreeJobsTableProjection,
) {
    let Some(projection) = (unsafe { projection.as_ref() }).copied() else {
        return;
    };
    unsafe { free_job_table_with_sync_projection(projection) };
}

/// Expand the C-owned MT job table when the requested worker count outgrows
/// its masked capacity. The table storage and custom allocator calls stay in
/// Rust; C supplies callbacks for the private descriptor synchronization
/// lifecycle.
#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_expandJobsTable(
    job_table_ptr: *mut *mut c_void,
    job_id_mask_ptr: *mut c_uint,
    nb_workers: c_uint,
    job_size: usize,
    custom_mem: ZstdCustomMem,
    init_sync: JobTableInitSync,
    destroy_sync: JobTableDestroySync,
) -> usize {
    unsafe {
        expand_job_table(
            job_table_ptr,
            job_id_mask_ptr,
            nb_workers,
            job_size,
            custom_mem,
            init_sync,
            destroy_sync,
        )
    }
}

unsafe fn create_buffer_pool(
    max_nb_buffers: usize,
    custom_mem: ZstdCustomMem,
) -> *mut RustBufferPool {
    if max_nb_buffers == 0 {
        return ptr::null_mut();
    }
    let Some(buffer_bytes) = checked_array_size::<MaybeUninit<ZstdMtBuffer>>(max_nb_buffers) else {
        return ptr::null_mut();
    };

    let pool = unsafe { custom_calloc(mem::size_of::<RustBufferPool>(), custom_mem) }
        .cast::<RustBufferPool>();
    if pool.is_null() {
        return ptr::null_mut();
    }
    let buffers =
        unsafe { custom_calloc(buffer_bytes, custom_mem) }.cast::<MaybeUninit<ZstdMtBuffer>>();
    if buffers.is_null() {
        unsafe { custom_free(pool.cast(), custom_mem) };
        return ptr::null_mut();
    }

    unsafe {
        pool.write(RustBufferPool {
            custom_mem,
            total_buffers: max_nb_buffers,
            buffers,
            state: Mutex::new(BufferPoolState {
                buffer_size: 64 << 10,
                nb_buffers: 0,
            }),
        });
    }
    pool
}

unsafe fn destroy_buffer_pool(pool: *mut RustBufferPool) {
    if pool.is_null() {
        return;
    }
    let custom_mem = unsafe { (*pool).custom_mem };
    let buffers = unsafe { (*pool).buffers };
    let total_buffers = unsafe { (*pool).total_buffers };
    for index in 0..total_buffers {
        let buffer = unsafe { buffers.add(index).read().assume_init() };
        unsafe { custom_free(buffer.start, custom_mem) };
    }
    unsafe { ptr::drop_in_place(pool) };
    unsafe {
        custom_free(buffers.cast(), custom_mem);
        custom_free(pool.cast(), custom_mem);
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_create(
    max_nb_buffers: c_uint,
    custom_mem: ZstdCustomMem,
) -> *mut RustBufferPool {
    unsafe { create_buffer_pool(max_nb_buffers as usize, custom_mem) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_free(pool: *mut RustBufferPool) {
    unsafe { destroy_buffer_pool(pool) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_sizeof(pool: *const RustBufferPool) -> usize {
    if pool.is_null() {
        return 0;
    }
    let pool_ref = unsafe { &*pool };
    let _state = pool_ref
        .state
        .lock()
        .unwrap_or_else(|error| error.into_inner());
    let mut total_buffer_size = 0usize;
    for index in 0..pool_ref.total_buffers {
        let buffer = unsafe { pool_ref.buffers.add(index).read().assume_init() };
        total_buffer_size = total_buffer_size.saturating_add(buffer.capacity);
    }
    mem::size_of::<RustBufferPool>()
        .saturating_add(
            pool_ref
                .total_buffers
                .saturating_mul(mem::size_of::<MaybeUninit<ZstdMtBuffer>>()),
        )
        .saturating_add(total_buffer_size)
}

#[inline]
fn sizeof_buffer_pool(wrapper_size: usize, rust_pool_size: usize) -> usize {
    wrapper_size.wrapping_add(rust_pool_size)
}

#[no_mangle]
pub extern "C" fn ZSTDMT_rust_sizeofBufferPool(wrapperSize: usize, rustPoolSize: usize) -> usize {
    sizeof_buffer_pool(wrapperSize, rustPoolSize)
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_set_size(
    pool: *mut RustBufferPool,
    buffer_size: usize,
) {
    if pool.is_null() {
        return;
    }
    let pool_ref = unsafe { &*pool };
    let mut state = pool_ref
        .state
        .lock()
        .unwrap_or_else(|error| error.into_inner());
    state.buffer_size = buffer_size;
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_expand(
    pool: *mut RustBufferPool,
    max_nb_buffers: c_uint,
) -> *mut RustBufferPool {
    if pool.is_null() {
        return ptr::null_mut();
    }
    let max_nb_buffers = max_nb_buffers as usize;
    let pool_ref = unsafe { &*pool };
    let (total_buffers, buffer_size, custom_mem) = {
        let state = pool_ref
            .state
            .lock()
            .unwrap_or_else(|error| error.into_inner());
        (
            pool_ref.total_buffers,
            state.buffer_size,
            pool_ref.custom_mem,
        )
    };
    if total_buffers >= max_nb_buffers {
        return pool;
    }

    // This matches the original resize contract: the old pool is consumed
    // before creating the larger replacement.
    unsafe { destroy_buffer_pool(pool) };
    let replacement = unsafe { create_buffer_pool(max_nb_buffers, custom_mem) };
    if !replacement.is_null() {
        unsafe { ZSTDMT_rust_buffer_pool_set_size(replacement, buffer_size) };
    }
    replacement
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_get(pool: *mut RustBufferPool) -> ZstdMtBuffer {
    if pool.is_null() {
        return ZstdMtBuffer::default();
    }
    let pool_ref = unsafe { &*pool };
    let (buffer_size, reusable) = {
        let mut state = pool_ref
            .state
            .lock()
            .unwrap_or_else(|error| error.into_inner());
        if state.buffer_size == 0 {
            return ZstdMtBuffer::default();
        }
        if state.nb_buffers == 0 {
            (state.buffer_size, ZstdMtBuffer::default())
        } else {
            state.nb_buffers -= 1;
            let index = state.nb_buffers;
            let buffer = unsafe { pool_ref.buffers.add(index).read().assume_init() };
            unsafe {
                pool_ref
                    .buffers
                    .add(index)
                    .write(MaybeUninit::new(ZstdMtBuffer::default()));
            }
            (state.buffer_size, buffer)
        }
    };

    if !reusable.start.is_null()
        && reusable.capacity >= buffer_size
        && (reusable.capacity >> 3) <= buffer_size
    {
        return reusable;
    }
    if !reusable.start.is_null() {
        unsafe { custom_free(reusable.start, pool_ref.custom_mem) };
    }

    let start = unsafe { custom_malloc(buffer_size, pool_ref.custom_mem) };
    ZstdMtBuffer {
        start,
        capacity: if start.is_null() { 0 } else { buffer_size },
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_release(
    pool: *mut RustBufferPool,
    buffer: ZstdMtBuffer,
) {
    if pool.is_null() || buffer.start.is_null() {
        return;
    }
    let pool_ref = unsafe { &*pool };
    let mut state = pool_ref
        .state
        .lock()
        .unwrap_or_else(|error| error.into_inner());
    if state.nb_buffers < pool_ref.total_buffers {
        let index = state.nb_buffers;
        unsafe {
            pool_ref.buffers.add(index).write(MaybeUninit::new(buffer));
        }
        state.nb_buffers += 1;
        return;
    }
    drop(state);
    unsafe { custom_free(buffer.start, pool_ref.custom_mem) };
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_buffer_pool_resize(
    pool: *mut RustBufferPool,
    buffer: ZstdMtBuffer,
) -> ZstdMtBuffer {
    if pool.is_null() || buffer.start.is_null() {
        return buffer;
    }
    let pool_ref = unsafe { &*pool };
    let buffer_size = {
        let state = pool_ref
            .state
            .lock()
            .unwrap_or_else(|error| error.into_inner());
        state.buffer_size
    };
    if buffer.capacity >= buffer_size {
        return buffer;
    }
    let start = unsafe { custom_malloc(buffer_size, pool_ref.custom_mem) };
    if start.is_null() {
        return buffer;
    }
    unsafe {
        ptr::copy_nonoverlapping(
            buffer.start.cast::<u8>(),
            start.cast::<u8>(),
            buffer.capacity,
        );
        custom_free(buffer.start, pool_ref.custom_mem);
    }
    ZstdMtBuffer {
        start,
        capacity: buffer_size,
    }
}

unsafe fn create_cctx_pool(nb_workers: usize, custom_mem: ZstdCustomMem) -> *mut RustCCtxPool {
    if nb_workers == 0 {
        return ptr::null_mut();
    }
    let Some(cctx_bytes) = checked_array_size::<MaybeUninit<*mut c_void>>(nb_workers) else {
        return ptr::null_mut();
    };
    let pool =
        unsafe { custom_calloc(mem::size_of::<RustCCtxPool>(), custom_mem) }.cast::<RustCCtxPool>();
    if pool.is_null() {
        return ptr::null_mut();
    }
    let cctxs = unsafe { custom_calloc(cctx_bytes, custom_mem) }.cast::<MaybeUninit<*mut c_void>>();
    if cctxs.is_null() {
        unsafe { custom_free(pool.cast(), custom_mem) };
        return ptr::null_mut();
    }
    let first = unsafe { ZSTD_createCCtx_advanced(custom_mem) };
    if first.is_null() {
        unsafe {
            custom_free(cctxs.cast(), custom_mem);
            custom_free(pool.cast(), custom_mem);
        }
        return ptr::null_mut();
    }
    unsafe { cctxs.write(MaybeUninit::new(first)) };
    unsafe {
        pool.write(RustCCtxPool {
            custom_mem,
            total_cctx: nb_workers,
            cctxs,
            state: Mutex::new(1),
        });
    }
    pool
}

unsafe fn destroy_cctx_pool(pool: *mut RustCCtxPool) {
    if pool.is_null() {
        return;
    }
    let custom_mem = unsafe { (*pool).custom_mem };
    let cctxs = unsafe { (*pool).cctxs };
    let total_cctx = unsafe { (*pool).total_cctx };
    for index in 0..total_cctx {
        let cctx = unsafe { cctxs.add(index).read().assume_init() };
        unsafe { ZSTD_freeCCtx(cctx) };
    }
    unsafe { ptr::drop_in_place(pool) };
    unsafe {
        custom_free(cctxs.cast(), custom_mem);
        custom_free(pool.cast(), custom_mem);
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_create(
    nb_workers: c_uint,
    custom_mem: ZstdCustomMem,
) -> *mut RustCCtxPool {
    unsafe { create_cctx_pool(nb_workers as usize, custom_mem) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_free(pool: *mut RustCCtxPool) {
    unsafe { destroy_cctx_pool(pool) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_sizeof(pool: *const RustCCtxPool) -> usize {
    if pool.is_null() {
        return 0;
    }
    let pool_ref = unsafe { &*pool };
    let _state = pool_ref
        .state
        .lock()
        .unwrap_or_else(|error| error.into_inner());
    let mut total_cctx_size = 0usize;
    for index in 0..pool_ref.total_cctx {
        let cctx = unsafe { pool_ref.cctxs.add(index).read().assume_init() };
        total_cctx_size =
            total_cctx_size.saturating_add(unsafe { ZSTD_sizeof_CCtx(cctx.cast_const()) });
    }
    mem::size_of::<RustCCtxPool>()
        .saturating_add(
            pool_ref
                .total_cctx
                .saturating_mul(mem::size_of::<MaybeUninit<*mut c_void>>()),
        )
        .saturating_add(total_cctx_size)
}

#[inline]
fn sizeof_cctx_pool(wrapper_size: usize, rust_pool_size: usize) -> usize {
    wrapper_size.wrapping_add(rust_pool_size)
}

#[no_mangle]
pub extern "C" fn ZSTDMT_rust_sizeofCCtxPool(wrapperSize: usize, rustPoolSize: usize) -> usize {
    sizeof_cctx_pool(wrapperSize, rustPoolSize)
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_expand(
    pool: *mut RustCCtxPool,
    nb_workers: c_uint,
) -> *mut RustCCtxPool {
    if pool.is_null() {
        return ptr::null_mut();
    }
    let nb_workers = nb_workers as usize;
    let pool_ref = unsafe { &*pool };
    if pool_ref.total_cctx >= nb_workers {
        return pool;
    }
    let custom_mem = pool_ref.custom_mem;
    unsafe { destroy_cctx_pool(pool) };
    unsafe { create_cctx_pool(nb_workers, custom_mem) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_get(pool: *mut RustCCtxPool) -> *mut c_void {
    if pool.is_null() {
        return ptr::null_mut();
    }
    let pool_ref = unsafe { &*pool };
    let cctx = {
        let mut available = pool_ref
            .state
            .lock()
            .unwrap_or_else(|error| error.into_inner());
        if *available == 0 {
            ptr::null_mut()
        } else {
            *available -= 1;
            let index = *available;
            unsafe { pool_ref.cctxs.add(index).read().assume_init() }
        }
    };
    if !cctx.is_null() {
        cctx
    } else {
        unsafe { ZSTD_createCCtx_advanced(pool_ref.custom_mem) }
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_release(pool: *mut RustCCtxPool, cctx: *mut c_void) {
    if pool.is_null() || cctx.is_null() {
        return;
    }
    let pool_ref = unsafe { &*pool };
    let mut available = pool_ref
        .state
        .lock()
        .unwrap_or_else(|error| error.into_inner());
    if *available < pool_ref.total_cctx {
        let index = *available;
        unsafe {
            pool_ref.cctxs.add(index).write(MaybeUninit::new(cctx));
        }
        *available += 1;
        return;
    }
    drop(available);
    unsafe { ZSTD_freeCCtx(cctx) };
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
    use std::{cell::RefCell, rc::Rc};

    const DEFAULT_MEM: ZstdCustomMem = ZstdCustomMem {
        customAlloc: None,
        customFree: None,
        opaque: ptr::null_mut(),
    };

    static JOB_TABLE_INIT_CALLS: AtomicUsize = AtomicUsize::new(0);
    static JOB_TABLE_DESTROY_CALLS: AtomicUsize = AtomicUsize::new(0);
    static JOB_TABLE_FAIL_INIT: AtomicBool = AtomicBool::new(false);
    static JOB_TABLE_TEST_LOCK: Mutex<()> = Mutex::new(());

    #[derive(Default)]
    struct FreeCCtxTestContext {
        events: Vec<&'static str>,
    }

    #[derive(Default)]
    struct CreateCCtxTestContext {
        events: Vec<&'static str>,
        workers: c_uint,
        jobs: c_uint,
        factory_ok: bool,
        resource_calls: usize,
    }

    fn record_create_cctx_event(context: *mut c_void, event: &'static str) {
        unsafe {
            (*context.cast::<CreateCCtxTestContext>())
                .events
                .push(event);
        }
    }

    unsafe extern "C" fn create_cctx_test_allocate(
        context: *mut c_void,
        _size: usize,
    ) -> *mut c_void {
        record_create_cctx_event(context, "allocate");
        ptr::dangling_mut::<c_void>()
    }

    unsafe extern "C" fn create_cctx_test_set_workers(
        context: *mut c_void,
        workers: c_uint,
    ) -> usize {
        record_create_cctx_event(context, "workers");
        unsafe { (*context.cast::<CreateCCtxTestContext>()).workers = workers };
        0
    }

    unsafe extern "C" fn create_cctx_test_initial_state(context: *mut c_void) {
        record_create_cctx_event(context, "initial-state");
    }

    unsafe extern "C" fn create_cctx_test_factory(
        context: *mut c_void,
        _workers: c_uint,
    ) -> *mut c_void {
        record_create_cctx_event(context, "factory");
        let context = unsafe { &*context.cast::<CreateCCtxTestContext>() };
        if context.factory_ok {
            ptr::dangling_mut::<c_void>()
        } else {
            ptr::null_mut()
        }
    }

    unsafe extern "C" fn create_cctx_test_jobs(
        context: *mut c_void,
        jobs: *mut c_uint,
    ) -> *mut c_void {
        record_create_cctx_event(context, "jobs");
        unsafe {
            (*context.cast::<CreateCCtxTestContext>()).jobs = 8;
            *jobs = 8;
        }
        ptr::dangling_mut::<c_void>()
    }

    unsafe extern "C" fn create_cctx_test_resource(
        context: *mut c_void,
        _workers: c_uint,
    ) -> *mut c_void {
        let event = unsafe {
            let state = &mut *context.cast::<CreateCCtxTestContext>();
            let event = match state.resource_calls {
                0 => "buffer-pool",
                1 => "cctx-pool",
                _ => "seq-pool",
            };
            state.resource_calls += 1;
            event
        };
        record_create_cctx_event(context, event);
        ptr::dangling_mut::<c_void>()
    }

    unsafe extern "C" fn create_cctx_test_serial_init(context: *mut c_void) -> c_int {
        record_create_cctx_event(context, "serial");
        0
    }

    unsafe extern "C" fn create_cctx_test_free(context: *mut c_void) {
        record_create_cctx_event(context, "free");
    }

    fn create_cctx_test_projection(
        context: &mut CreateCCtxTestContext,
        custom_mem: ZstdCustomMem,
    ) -> ZSTDMT_RustCreateCCtxProjection {
        ZSTDMT_RustCreateCCtxProjection {
            callback_context: context as *mut CreateCCtxTestContext as *mut c_void,
            requested_nb_workers: 99,
            max_nb_workers: 4,
            context_size: 128,
            custom_mem,
        }
    }

    #[derive(Default)]
    struct ResizeTestContext {
        events: Vec<&'static str>,
        resource_calls: usize,
        fail: Option<&'static str>,
    }

    fn record_resize_event(context: *mut c_void, event: &'static str) -> bool {
        let state = unsafe { &mut *context.cast::<ResizeTestContext>() };
        state.events.push(event);
        state.fail == Some(event)
    }

    unsafe extern "C" fn resize_test_step(context: *mut c_void, _workers: c_uint) -> usize {
        let state = unsafe { &*context.cast::<ResizeTestContext>() };
        let event = match state.events.len() {
            0 => "factory",
            _ => "jobs",
        };
        if record_resize_event(context, event) {
            ERROR(ZstdErrorCode::MemoryAllocation)
        } else {
            0
        }
    }

    unsafe extern "C" fn resize_test_resource(
        context: *mut c_void,
        _workers: c_uint,
    ) -> *mut c_void {
        let state = unsafe { &mut *context.cast::<ResizeTestContext>() };
        let event = match state.resource_calls {
            0 => "buffer-pool",
            1 => "cctx-pool",
            _ => "seq-pool",
        };
        state.resource_calls += 1;
        if record_resize_event(context, event) {
            ptr::null_mut()
        } else {
            ptr::dangling_mut::<c_void>()
        }
    }

    unsafe extern "C" fn resize_test_set_workers(context: *mut c_void, _workers: c_uint) -> usize {
        assert!(!record_resize_event(context, "set-workers"));
        0
    }

    fn resize_test_projection(
        context: &mut ResizeTestContext,
        workers: c_uint,
    ) -> ZSTDMT_RustResizeProjection {
        ZSTDMT_RustResizeProjection {
            callback_context: context as *mut ResizeTestContext as *mut c_void,
            nb_workers: workers,
        }
    }

    #[test]
    fn resize_preserves_resource_order_and_publishes_workers_last() {
        let mut context = ResizeTestContext::default();
        let projection = resize_test_projection(&mut context, 6);
        let result = unsafe {
            ZSTDMT_rust_resize(
                &projection,
                Some(resize_test_step),
                Some(resize_test_step),
                Some(resize_test_resource),
                Some(resize_test_resource),
                Some(resize_test_resource),
                Some(resize_test_set_workers),
            )
        };

        assert_eq!(result, 0);
        assert_eq!(
            context.events,
            [
                "factory",
                "jobs",
                "buffer-pool",
                "cctx-pool",
                "seq-pool",
                "set-workers",
            ]
        );
    }

    #[test]
    fn resize_stops_after_resource_failure_without_publishing_workers() {
        let mut context = ResizeTestContext {
            fail: Some("cctx-pool"),
            ..ResizeTestContext::default()
        };
        let projection = resize_test_projection(&mut context, 3);
        let result = unsafe {
            ZSTDMT_rust_resize(
                &projection,
                Some(resize_test_step),
                Some(resize_test_step),
                Some(resize_test_resource),
                Some(resize_test_resource),
                Some(resize_test_resource),
                Some(resize_test_set_workers),
            )
        };

        assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation));
        assert_eq!(
            context.events,
            ["factory", "jobs", "buffer-pool", "cctx-pool"]
        );
    }

    fn record_free_cctx_event(context: *mut c_void, event: &'static str) {
        unsafe {
            (*context.cast::<FreeCCtxTestContext>()).events.push(event);
        }
    }

    unsafe extern "C" fn free_cctx_test_factory(context: *mut c_void) {
        record_free_cctx_event(context, "factory");
    }

    unsafe extern "C" fn free_cctx_test_release_jobs(context: *mut c_void) {
        record_free_cctx_event(context, "release-jobs");
    }

    unsafe extern "C" fn free_cctx_test_jobs(context: *mut c_void) {
        record_free_cctx_event(context, "jobs");
    }

    unsafe extern "C" fn free_cctx_test_buffer_pool(context: *mut c_void) {
        record_free_cctx_event(context, "buffer-pool");
    }

    unsafe extern "C" fn free_cctx_test_cctx_pool(context: *mut c_void) {
        record_free_cctx_event(context, "cctx-pool");
    }

    unsafe extern "C" fn free_cctx_test_seq_pool(context: *mut c_void) {
        record_free_cctx_event(context, "seq-pool");
    }

    unsafe extern "C" fn free_cctx_test_serial_state(context: *mut c_void) {
        record_free_cctx_event(context, "serial-state");
    }

    unsafe extern "C" fn free_cctx_test_cdict(context: *mut c_void) {
        record_free_cctx_event(context, "cdict");
    }

    unsafe extern "C" fn free_cctx_test_round_buffer(context: *mut c_void) {
        record_free_cctx_event(context, "round-buffer");
    }

    unsafe extern "C" fn free_cctx_test_mtctx(context: *mut c_void) {
        record_free_cctx_event(context, "mtctx");
    }

    fn free_cctx_test_state(
        context: &mut FreeCCtxTestContext,
        provided_factory: usize,
        round_buffer_present: usize,
    ) -> ZSTDMT_RustFreeCCtxState {
        ZSTDMT_RustFreeCCtxState {
            callback_context: context as *mut FreeCCtxTestContext as *mut c_void,
            provided_factory,
            round_buffer_present,
            free_factory: Some(free_cctx_test_factory),
            release_all_job_resources: Some(free_cctx_test_release_jobs),
            free_jobs: Some(free_cctx_test_jobs),
            free_buffer_pool: Some(free_cctx_test_buffer_pool),
            free_cctx_pool: Some(free_cctx_test_cctx_pool),
            free_seq_pool: Some(free_cctx_test_seq_pool),
            free_serial_state: Some(free_cctx_test_serial_state),
            free_cdict: Some(free_cctx_test_cdict),
            free_round_buffer: Some(free_cctx_test_round_buffer),
            free_mtctx: Some(free_cctx_test_mtctx),
        }
    }

    #[test]
    fn free_cctx_null_state_returns_zero_without_callbacks() {
        assert_eq!(unsafe { ZSTDMT_rust_freeCCtx(ptr::null()) }, 0);
    }

    #[test]
    fn create_cctx_preserves_resource_order_and_clamps_workers() {
        let mut context = CreateCCtxTestContext {
            factory_ok: true,
            ..CreateCCtxTestContext::default()
        };
        let projection = create_cctx_test_projection(&mut context, DEFAULT_MEM);

        let result = unsafe {
            ZSTDMT_rust_createCCtx(
                &projection,
                Some(create_cctx_test_allocate),
                Some(create_cctx_test_set_workers),
                Some(create_cctx_test_initial_state),
                Some(create_cctx_test_factory),
                Some(create_cctx_test_jobs),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_serial_init),
                Some(create_cctx_test_free),
            )
        };

        assert_eq!(result, ptr::dangling_mut::<c_void>());
        assert_eq!(context.workers, 4);
        assert_eq!(context.jobs, 8);
        assert_eq!(
            context.events,
            vec![
                "allocate",
                "workers",
                "initial-state",
                "factory",
                "jobs",
                "buffer-pool",
                "cctx-pool",
                "seq-pool",
                "serial",
            ]
        );
    }

    #[test]
    fn create_cctx_runs_remaining_callbacks_before_failure_cleanup() {
        let mut context = CreateCCtxTestContext::default();
        let projection = create_cctx_test_projection(&mut context, DEFAULT_MEM);

        let result = unsafe {
            ZSTDMT_rust_createCCtx(
                &projection,
                Some(create_cctx_test_allocate),
                Some(create_cctx_test_set_workers),
                Some(create_cctx_test_initial_state),
                Some(create_cctx_test_factory),
                Some(create_cctx_test_jobs),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_serial_init),
                Some(create_cctx_test_free),
            )
        };

        assert!(result.is_null());
        assert_eq!(
            context.events,
            vec![
                "allocate",
                "workers",
                "initial-state",
                "factory",
                "jobs",
                "buffer-pool",
                "cctx-pool",
                "seq-pool",
                "serial",
                "free",
            ]
        );
    }

    unsafe extern "C" fn create_cctx_test_alloc(_opaque: *mut c_void, _size: usize) -> *mut c_void {
        ptr::null_mut()
    }

    #[test]
    fn create_cctx_rejects_mismatched_custom_allocator() {
        let mut context = CreateCCtxTestContext {
            factory_ok: true,
            ..CreateCCtxTestContext::default()
        };
        let invalid_mem = ZstdCustomMem {
            customAlloc: Some(create_cctx_test_alloc),
            customFree: None,
            opaque: ptr::null_mut(),
        };
        let projection = create_cctx_test_projection(&mut context, invalid_mem);

        let result = unsafe {
            ZSTDMT_rust_createCCtx(
                &projection,
                Some(create_cctx_test_allocate),
                Some(create_cctx_test_set_workers),
                Some(create_cctx_test_initial_state),
                Some(create_cctx_test_factory),
                Some(create_cctx_test_jobs),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_resource),
                Some(create_cctx_test_serial_init),
                Some(create_cctx_test_free),
            )
        };

        assert!(result.is_null());
        assert!(context.events.is_empty());
    }

    #[test]
    fn free_cctx_preserves_callback_order_and_ownership_conditions() {
        let mut context = FreeCCtxTestContext::default();
        let state = free_cctx_test_state(&mut context, 0, 1);

        assert_eq!(unsafe { ZSTDMT_rust_freeCCtx(&state) }, 0);
        assert_eq!(
            context.events,
            vec![
                "factory",
                "release-jobs",
                "jobs",
                "buffer-pool",
                "cctx-pool",
                "seq-pool",
                "serial-state",
                "cdict",
                "round-buffer",
                "mtctx",
            ]
        );

        context.events.clear();
        let state = free_cctx_test_state(&mut context, 1, 0);
        assert_eq!(unsafe { ZSTDMT_rust_freeCCtx(&state) }, 0);
        assert_eq!(
            context.events,
            vec![
                "release-jobs",
                "jobs",
                "buffer-pool",
                "cctx-pool",
                "seq-pool",
                "serial-state",
                "cdict",
                "mtctx",
            ]
        );
    }

    struct MockChunkCompressor {
        continue_inputs: Vec<usize>,
        end_inputs: Vec<usize>,
        continue_results: Vec<usize>,
        end_results: Vec<usize>,
    }

    #[derive(Default)]
    struct MockChunkProgress {
        calls: Vec<(usize, usize)>,
    }

    #[derive(Default)]
    struct MockJobProgress {
        events: Vec<&'static str>,
        debug_calls: Vec<(usize, usize)>,
    }

    #[derive(Default)]
    struct MockCompressionJob {
        events: Vec<&'static str>,
        c_size: usize,
        consumed_size: usize,
        expected_src_size: usize,
    }

    struct WaitForLdmTestContext {
        events: Vec<&'static str>,
        overlaps: Vec<bool>,
        next_overlap: usize,
    }

    struct SerialWaitForTurnTestContext {
        events: Vec<&'static str>,
        next_job_id: c_uint,
        wake_job_id: c_uint,
    }

    unsafe extern "C" fn serial_wait_for_turn_test_lock(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialWaitForTurnTestContext>())
                .events
                .push("lock")
        };
    }

    unsafe extern "C" fn serial_wait_for_turn_test_wait(context: *mut c_void) {
        unsafe {
            let context = &mut *context.cast::<SerialWaitForTurnTestContext>();
            context.events.push("wait");
            context.next_job_id = context.wake_job_id;
        }
    }

    fn serial_wait_for_turn_test_state(
        context: &mut SerialWaitForTurnTestContext,
        job_id: c_uint,
    ) -> ZSTDMT_RustSerialWaitForTurnState {
        ZSTDMT_RustSerialWaitForTurnState {
            callback_context: context as *mut _ as *mut c_void,
            next_job_id: &mut context.next_job_id,
            lock: Some(serial_wait_for_turn_test_lock),
            wait: Some(serial_wait_for_turn_test_wait),
            job_id,
        }
    }

    struct SerialAdvanceTestContext {
        events: Vec<&'static str>,
    }

    unsafe extern "C" fn serial_advance_test_broadcast(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialAdvanceTestContext>())
                .events
                .push("broadcast")
        };
    }

    unsafe extern "C" fn serial_advance_test_unlock(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialAdvanceTestContext>())
                .events
                .push("unlock")
        };
    }

    fn serial_advance_test_state(
        context: &mut SerialAdvanceTestContext,
        next_job_id: &mut c_uint,
    ) -> ZSTDMT_RustSerialAdvanceState {
        ZSTDMT_RustSerialAdvanceState {
            callback_context: context as *mut _ as *mut c_void,
            next_job_id,
            broadcast: Some(serial_advance_test_broadcast),
            unlock: Some(serial_advance_test_unlock),
        }
    }

    struct SerialEnsureFinishedTestContext {
        events: Vec<&'static str>,
    }

    unsafe extern "C" fn serial_ensure_finished_test_lock(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialEnsureFinishedTestContext>())
                .events
                .push("lock")
        };
    }

    unsafe extern "C" fn serial_ensure_finished_test_broadcast(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialEnsureFinishedTestContext>())
                .events
                .push("broadcast")
        };
    }

    unsafe extern "C" fn serial_ensure_finished_test_ldm_lock(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialEnsureFinishedTestContext>())
                .events
                .push("ldm_lock")
        };
    }

    unsafe extern "C" fn serial_ensure_finished_test_clear_ldm(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialEnsureFinishedTestContext>())
                .events
                .push("clear_ldm")
        };
    }

    unsafe extern "C" fn serial_ensure_finished_test_ldm_signal(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialEnsureFinishedTestContext>())
                .events
                .push("ldm_signal")
        };
    }

    unsafe extern "C" fn serial_ensure_finished_test_ldm_unlock(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialEnsureFinishedTestContext>())
                .events
                .push("ldm_unlock")
        };
    }

    unsafe extern "C" fn serial_ensure_finished_test_unlock(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialEnsureFinishedTestContext>())
                .events
                .push("unlock")
        };
    }

    unsafe extern "C" fn serial_ensure_finished_test_on_skip(
        context: *mut c_void,
        job_id: c_uint,
        c_size: usize,
    ) {
        assert_eq!(job_id, 4);
        assert_eq!(c_size, 123);
        unsafe {
            (*context.cast::<SerialEnsureFinishedTestContext>())
                .events
                .push("skip")
        };
    }

    fn serial_ensure_finished_test_state(
        context: &mut SerialEnsureFinishedTestContext,
        next_job_id: &mut c_uint,
        job_id: c_uint,
        c_size: usize,
    ) -> ZSTDMT_RustSerialEnsureFinishedState {
        ZSTDMT_RustSerialEnsureFinishedState {
            callback_context: context as *mut _ as *mut c_void,
            next_job_id,
            lock: Some(serial_ensure_finished_test_lock),
            broadcast: Some(serial_ensure_finished_test_broadcast),
            ldm_lock: Some(serial_ensure_finished_test_ldm_lock),
            clear_ldm_window: Some(serial_ensure_finished_test_clear_ldm),
            ldm_signal: Some(serial_ensure_finished_test_ldm_signal),
            ldm_unlock: Some(serial_ensure_finished_test_ldm_unlock),
            unlock: Some(serial_ensure_finished_test_unlock),
            on_skip: Some(serial_ensure_finished_test_on_skip),
            c_size,
            job_id,
        }
    }

    unsafe extern "C" fn wait_for_ldm_test_lock(context: *mut c_void) {
        unsafe {
            (*context.cast::<WaitForLdmTestContext>())
                .events
                .push("lock")
        };
    }

    unsafe extern "C" fn wait_for_ldm_test_overlaps(
        context: *mut c_void,
        _buffer_start: *mut c_void,
        _buffer_capacity: usize,
    ) -> c_int {
        let context = unsafe { &mut *context.cast::<WaitForLdmTestContext>() };
        context.events.push("overlap");
        let overlaps = context
            .overlaps
            .get(context.next_overlap)
            .copied()
            .unwrap_or(false);
        context.next_overlap += 1;
        c_int::from(overlaps)
    }

    unsafe extern "C" fn wait_for_ldm_test_wait(context: *mut c_void) {
        unsafe {
            (*context.cast::<WaitForLdmTestContext>())
                .events
                .push("wait")
        };
    }

    unsafe extern "C" fn wait_for_ldm_test_unlock(context: *mut c_void) {
        unsafe {
            (*context.cast::<WaitForLdmTestContext>())
                .events
                .push("unlock")
        };
    }

    fn wait_for_ldm_test_state(
        context: &mut WaitForLdmTestContext,
        ldm_enabled: c_int,
    ) -> ZSTDMT_RustWaitForLdmState {
        ZSTDMT_RustWaitForLdmState {
            callback_context: context as *mut _ as *mut c_void,
            buffer_start: ptr::null_mut(),
            buffer_capacity: 8,
            ldm_enabled,
            lock: Some(wait_for_ldm_test_lock),
            overlaps: Some(wait_for_ldm_test_overlaps),
            wait: Some(wait_for_ldm_test_wait),
            unlock: Some(wait_for_ldm_test_unlock),
        }
    }

    #[test]
    fn wait_for_ldm_runs_overlap_wait_loop_before_unlocking() {
        let mut context = WaitForLdmTestContext {
            events: Vec::new(),
            overlaps: vec![true, false],
            next_overlap: 0,
        };
        let state = wait_for_ldm_test_state(&mut context, ZSTD_PS_ENABLE);

        unsafe { ZSTDMT_rust_waitForLdmComplete(&state) };

        assert_eq!(
            context.events,
            vec!["lock", "overlap", "wait", "overlap", "unlock"]
        );
        assert_eq!(context.next_overlap, 2);
    }

    #[test]
    fn wait_for_ldm_skips_callbacks_when_ldm_is_disabled() {
        let mut context = WaitForLdmTestContext {
            events: Vec::new(),
            overlaps: vec![true],
            next_overlap: 0,
        };
        let state = wait_for_ldm_test_state(&mut context, 0);

        unsafe { ZSTDMT_rust_waitForLdmComplete(&state) };

        assert!(context.events.is_empty());
        assert_eq!(context.next_overlap, 0);
    }

    #[test]
    fn serial_wait_for_turn_waits_until_job_is_ready() {
        let mut context = SerialWaitForTurnTestContext {
            events: Vec::new(),
            next_job_id: 2,
            wake_job_id: 4,
        };
        let state = serial_wait_for_turn_test_state(&mut context, 4);

        let result = unsafe { ZSTDMT_rust_serialStateWaitForTurn(&state) };

        assert_eq!(result, 1);
        assert_eq!(context.events, vec!["lock", "wait"]);
        assert_eq!(context.next_job_id, 4);
    }

    #[test]
    fn serial_wait_for_turn_reports_skip_without_waiting_for_later_job() {
        let mut context = SerialWaitForTurnTestContext {
            events: Vec::new(),
            next_job_id: 5,
            wake_job_id: 5,
        };
        let state = serial_wait_for_turn_test_state(&mut context, 4);

        let result = unsafe { ZSTDMT_rust_serialStateWaitForTurn(&state) };

        assert_eq!(result, 0);
        assert_eq!(context.events, vec!["lock"]);
        assert_eq!(context.next_job_id, 5);
    }

    #[test]
    fn serial_advance_increments_before_broadcast_and_unlock() {
        let mut context = SerialAdvanceTestContext { events: Vec::new() };
        let mut next_job_id = 7;
        let state = serial_advance_test_state(&mut context, &mut next_job_id);

        unsafe { ZSTDMT_rust_serialStateAdvance(&state) };

        assert_eq!(next_job_id, 8);
        assert_eq!(context.events, vec!["broadcast", "unlock"]);
    }

    #[test]
    fn serial_advance_wraps_the_job_counter_like_c_unsigned_arithmetic() {
        let mut context = SerialAdvanceTestContext { events: Vec::new() };
        let mut next_job_id = c_uint::MAX;
        let state = serial_advance_test_state(&mut context, &mut next_job_id);

        unsafe { ZSTDMT_rust_serialStateAdvance(&state) };

        assert_eq!(next_job_id, 0);
        assert_eq!(context.events, vec!["broadcast", "unlock"]);
    }

    #[test]
    fn serial_ensure_finished_runs_failed_job_cleanup_in_order() {
        let mut context = SerialEnsureFinishedTestContext { events: Vec::new() };
        let mut next_job_id = 4;
        let state = serial_ensure_finished_test_state(&mut context, &mut next_job_id, 4, 123);

        unsafe { ZSTDMT_rust_serialStateEnsureFinishedOrchestrated(&state) };

        assert_eq!(next_job_id, 5);
        assert_eq!(
            context.events,
            vec![
                "lock",
                "skip",
                "broadcast",
                "ldm_lock",
                "clear_ldm",
                "ldm_signal",
                "ldm_unlock",
                "unlock"
            ]
        );
    }

    #[test]
    fn serial_ensure_finished_unlocks_without_cleanup_for_later_job() {
        let mut context = SerialEnsureFinishedTestContext { events: Vec::new() };
        let mut next_job_id = 6;
        let state = serial_ensure_finished_test_state(&mut context, &mut next_job_id, 4, 123);

        unsafe { ZSTDMT_rust_serialStateEnsureFinishedOrchestrated(&state) };

        assert_eq!(next_job_id, 6);
        assert_eq!(context.events, vec!["lock", "unlock"]);
    }

    fn record_compression_job_event(state: &Rc<RefCell<MockCompressionJob>>, event: &'static str) {
        state.borrow_mut().events.push(event);
    }

    fn record_compression_job_finish_event(context: *mut c_void, event: &'static str) {
        unsafe {
            (*context.cast::<RefCell<MockCompressionJob>>())
                .borrow_mut()
                .events
                .push(event);
        }
    }

    unsafe extern "C" fn compression_job_finish_test_ensure(context: *mut c_void) {
        record_compression_job_finish_event(context, "ensure");
    }

    unsafe extern "C" fn compression_job_finish_test_release_seq(context: *mut c_void) {
        record_compression_job_finish_event(context, "release-seq");
    }

    unsafe extern "C" fn compression_job_finish_test_release_cctx(context: *mut c_void) {
        record_compression_job_finish_event(context, "release-cctx");
    }

    unsafe extern "C" fn compression_job_finish_test_lock(context: *mut c_void) {
        record_compression_job_finish_event(context, "lock");
    }

    unsafe extern "C" fn compression_job_finish_test_signal(context: *mut c_void) {
        record_compression_job_finish_event(context, "signal");
    }

    unsafe extern "C" fn compression_job_finish_test_unlock(context: *mut c_void) {
        record_compression_job_finish_event(context, "unlock");
    }

    unsafe extern "C" fn compression_job_finish_test_validate(
        context: *mut c_void,
        src_size: usize,
    ) {
        let state = unsafe { &*context.cast::<RefCell<MockCompressionJob>>() };
        assert_eq!(state.borrow().expected_src_size, src_size);
        record_compression_job_finish_event(context, "validate");
    }

    fn compression_job_finish_test_projection(
        state: &Rc<RefCell<MockCompressionJob>>,
        src_size: usize,
    ) -> ZSTDMT_compressionJobFinishProjection {
        let state_cell = Rc::as_ptr(state);
        let state_value = unsafe { (*state_cell).as_ptr() };
        let (c_size, consumed) = unsafe {
            (*state_value).expected_src_size = src_size;
            (
                &mut (*state_value).c_size as *mut usize,
                &mut (*state_value).consumed_size as *mut usize,
            )
        };
        let callback_context = state_cell.cast_mut().cast();
        ZSTDMT_compressionJobFinishProjection {
            callbackContext: callback_context,
            srcSize: src_size,
            ensureFinished: Some(compression_job_finish_test_ensure),
            publication: ZSTDMT_compressionJobFinishPublicationProjection {
                callbackContext: callback_context,
                cSize: c_size,
                consumed,
                srcSize: src_size,
                lock: Some(compression_job_finish_test_lock),
                signal: Some(compression_job_finish_test_signal),
                unlock: Some(compression_job_finish_test_unlock),
                validateConsumed: Some(compression_job_finish_test_validate),
            },
            releaseSeq: Some(compression_job_finish_test_release_seq),
            releaseCCtx: Some(compression_job_finish_test_release_cctx),
        }
    }

    #[test]
    fn compression_job_frame_header_projection_writes_header_and_clears_repcodes() {
        let mut stage = ZSTD_COMPRESSION_STAGE_INIT;
        let mut rep_codes = [11u32, 22, 33];
        let mut output = [0xa5u8; 18];
        let state = ZSTDMT_compressionJobFrameHeaderState {
            dst: output.as_mut_ptr().cast(),
            dst_capacity: output.len(),
            stage: &mut stage,
            no_dict_id_flag: 0,
            checksum_flag: 0,
            content_size_flag: 1,
            format: 0,
            window_log: 20,
            pledged_src_size_plus_one: 1,
            dict_id: 0,
            rep_codes: rep_codes.as_mut_ptr(),
        };

        let mut created_stage = ZSTD_COMPRESSION_STAGE_CREATED;
        let created_state = ZSTDMT_compressionJobFrameHeaderState {
            stage: &mut created_stage,
            ..state
        };
        let created = unsafe { ZSTDMT_rust_writeFrameHeader(&created_state) };
        assert_eq!(created, ERROR(ZstdErrorCode::StageWrong));
        assert_eq!(created_stage, ZSTD_COMPRESSION_STAGE_CREATED);
        assert_eq!(rep_codes, [11, 22, 33]);
        let result = unsafe { ZSTDMT_rust_writeFrameHeader(&state) };

        assert_eq!(result, 6);
        assert_eq!(stage, ZSTD_COMPRESSION_STAGE_ONGOING);
        assert_eq!(rep_codes, [0; 3]);
        assert_eq!(&output[..result], &[0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x00]);
        assert_eq!(&output[result..], &[0xa5; 12]);
    }

    #[test]
    fn compression_job_sequence_gate_skips_empty_store_without_ldm_assertion() {
        let mut calls = Vec::new();
        let projection = ZSTDMT_compressionJobSequenceProjection {
            cctx: 0x10usize as *mut c_void,
            sequences: ptr::null_mut(),
            nbSequences: 0,
            ldmEnabled: 0,
        };

        apply_compression_job_sequences_with(Some(projection), |cctx, sequences, count| {
            calls.push((cctx, sequences, count));
        });

        assert!(calls.is_empty());
    }

    #[test]
    fn compression_job_sequence_gate_forwards_nonempty_store_after_ldm_check() {
        let mut calls = Vec::new();
        let projection = ZSTDMT_compressionJobSequenceProjection {
            cctx: 0x10usize as *mut c_void,
            sequences: 0x20usize as *mut c_void,
            nbSequences: 3,
            ldmEnabled: ZSTD_PS_ENABLE,
        };

        apply_compression_job_sequences_with(Some(projection), |cctx, sequences, count| {
            calls.push((cctx, sequences, count));
        });

        assert_eq!(
            calls,
            vec![(
                0x10usize as *mut c_void,
                0x20usize as *mut c_void,
                3
            )]
        );
    }

    #[test]
    fn compression_job_resources_order_pool_gets_destination_and_ldm_check() {
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let acquire_cctx_events = Rc::clone(&events);
        let acquire_seq_events = Rc::clone(&events);
        let cctx_ready_events = Rc::clone(&events);
        let destination_events = Rc::clone(&events);
        let seq_ready_events = Rc::clone(&events);
        let publish_events = Rc::clone(&events);

        let result = compression_job_acquire_resources_with(
            ZSTDMT_compressionJobResourceProjection {
                ldmEnabled: 1,
                hasDestination: 0,
            },
            move || acquire_cctx_events.borrow_mut().push("cctx"),
            move || acquire_seq_events.borrow_mut().push("seq"),
            move || {
                cctx_ready_events.borrow_mut().push("cctx-ready");
                true
            },
            move || {
                destination_events.borrow_mut().push("destination");
                0
            },
            move || {
                seq_ready_events.borrow_mut().push("seq-ready");
                true
            },
            move || publish_events.borrow_mut().push("publish"),
        );

        assert_eq!(result, 0);
        assert_eq!(
            &*events.borrow(),
            &[
                "cctx",
                "seq",
                "cctx-ready",
                "destination",
                "publish",
                "seq-ready",
            ]
        );
    }

    #[test]
    fn compression_job_resources_reject_missing_cctx_after_both_pool_gets() {
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let acquire_cctx_events = Rc::clone(&events);
        let acquire_seq_events = Rc::clone(&events);
        let cctx_ready_events = Rc::clone(&events);
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);

        let result = compression_job_acquire_resources_with(
            ZSTDMT_compressionJobResourceProjection {
                ldmEnabled: 1,
                hasDestination: 0,
            },
            move || acquire_cctx_events.borrow_mut().push("cctx"),
            move || acquire_seq_events.borrow_mut().push("seq"),
            move || {
                cctx_ready_events.borrow_mut().push("cctx-ready");
                false
            },
            || panic!("CCtx failure must stop before destination acquisition"),
            || panic!("CCtx failure must stop before LDM validation"),
            || panic!("CCtx failure must stop before destination publication"),
        );

        assert_eq!(result, expected_error);
        assert_eq!(&*events.borrow(), &["cctx", "seq", "cctx-ready"]);
    }

    #[test]
    fn compression_job_resources_stops_before_publication_on_destination_error() {
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let acquire_cctx_events = Rc::clone(&events);
        let acquire_seq_events = Rc::clone(&events);
        let cctx_ready_events = Rc::clone(&events);
        let destination_events = Rc::clone(&events);
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);

        let result = compression_job_acquire_resources_with(
            ZSTDMT_compressionJobResourceProjection {
                ldmEnabled: 1,
                hasDestination: 0,
            },
            move || acquire_cctx_events.borrow_mut().push("cctx"),
            move || acquire_seq_events.borrow_mut().push("seq"),
            move || {
                cctx_ready_events.borrow_mut().push("cctx-ready");
                true
            },
            move || {
                destination_events.borrow_mut().push("destination");
                expected_error
            },
            || panic!("destination failure must stop before LDM validation"),
            || panic!("destination failure must stop before publication"),
        );

        assert_eq!(result, expected_error);
        assert_eq!(
            &*events.borrow(),
            &["cctx", "seq", "cctx-ready", "destination"]
        );
    }

    #[test]
    fn compression_job_resources_publishes_before_missing_ldm_store_error() {
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let acquire_cctx_events = Rc::clone(&events);
        let acquire_seq_events = Rc::clone(&events);
        let cctx_ready_events = Rc::clone(&events);
        let seq_ready_events = Rc::clone(&events);
        let publish_events = Rc::clone(&events);
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);

        let result = compression_job_acquire_resources_with(
            ZSTDMT_compressionJobResourceProjection {
                ldmEnabled: 1,
                hasDestination: 1,
            },
            move || acquire_cctx_events.borrow_mut().push("cctx"),
            move || acquire_seq_events.borrow_mut().push("seq"),
            move || {
                cctx_ready_events.borrow_mut().push("cctx-ready");
                true
            },
            || panic!("an existing destination must skip buffer acquisition"),
            move || {
                seq_ready_events.borrow_mut().push("seq-ready");
                false
            },
            move || publish_events.borrow_mut().push("publish"),
        );

        assert_eq!(result, expected_error);
        assert_eq!(
            &*events.borrow(),
            &["cctx", "seq", "cctx-ready", "publish", "seq-ready"]
        );
    }

    #[test]
    fn compression_job_trace_guards_only_continuation_jobs() {
        let first_job_events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let first_job_assert_events = Rc::clone(&first_job_events);
        let first_job_trace_events = Rc::clone(&first_job_events);
        compression_job_trace_with(
            1,
            move || first_job_assert_events.borrow_mut().push("assert"),
            move || first_job_trace_events.borrow_mut().push("trace"),
        );
        assert_eq!(&*first_job_events.borrow(), &["trace"]);

        let continuation_events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let continuation_assert_events = Rc::clone(&continuation_events);
        let continuation_trace_events = Rc::clone(&continuation_events);
        compression_job_trace_with(
            0,
            move || continuation_assert_events.borrow_mut().push("assert"),
            move || continuation_trace_events.borrow_mut().push("trace"),
        );
        assert_eq!(&*continuation_events.borrow(), &["assert", "trace"]);
    }

    #[test]
    fn compression_job_runs_first_job_stages_and_reports_final_block() {
        let state = Rc::new(RefCell::new(MockCompressionJob::default()));
        let acquire_state = Rc::clone(&state);
        let prepare_state = Rc::clone(&state);
        let sequence_state = Rc::clone(&state);
        let begin_state = Rc::clone(&state);
        let apply_state = Rc::clone(&state);
        let compress_state = Rc::clone(&state);
        let trace_state = Rc::clone(&state);
        let finish = compression_job_finish_test_projection(&state, 19);

        compression_job_with(
            ZSTDMT_compressionJobProjection {
                firstJob: 1,
                lastJob: 1,
                frameHeaderState: ptr::null(),
                sequenceState: ptr::null(),
            },
            finish,
            move || {
                record_compression_job_event(&acquire_state, "acquire");
                0
            },
            move || record_compression_job_event(&prepare_state, "prepare"),
            move || record_compression_job_event(&sequence_state, "sequences"),
            move || {
                record_compression_job_event(&begin_state, "begin");
                0
            },
            move || record_compression_job_event(&apply_state, "apply"),
            || panic!("first jobs do not rewrite a frame header"),
            move |last_job| {
                assert_eq!(last_job, 1);
                record_compression_job_event(&compress_state, "compress");
                ZSTDMT_chunkProcessResult {
                    error: 0,
                    lastBlockSize: 7,
                }
            },
            move || record_compression_job_event(&trace_state, "trace"),
        );

        let state = state.borrow();
        assert_eq!(
            state.events,
            vec![
                "acquire",
                "prepare",
                "sequences",
                "begin",
                "apply",
                "compress",
                "trace",
                "ensure",
                "release-seq",
                "release-cctx",
                "lock",
                "unlock",
                "validate",
                "lock",
                "unlock",
                "lock",
                "signal",
                "unlock",
            ]
        );
        assert_eq!(state.c_size, 7);
        assert_eq!(state.consumed_size, 19);
    }

    #[test]
    fn compression_job_normalizes_resource_error_before_finish_cleanup() {
        let state = Rc::new(RefCell::new(MockCompressionJob::default()));
        let acquire_state = Rc::clone(&state);
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
        let finish = compression_job_finish_test_projection(&state, 11);

        compression_job_with(
            ZSTDMT_compressionJobProjection {
                firstJob: 1,
                lastJob: 0,
                frameHeaderState: ptr::null(),
                sequenceState: ptr::null(),
            },
            finish,
            move || {
                record_compression_job_event(&acquire_state, "acquire");
                expected_error
            },
            || panic!("resource failure must skip parameter preparation"),
            || panic!("resource failure must skip sequence generation"),
            || panic!("resource failure must skip context initialization"),
            || panic!("resource failure must skip sequence application"),
            || panic!("resource failure must skip frame-header writing"),
            |_| panic!("resource failure must skip compression"),
            || panic!("resource failure must skip tracing"),
        );

        let state = state.borrow();
        assert_eq!(
            state.events,
            vec![
                "acquire",
                "lock",
                "unlock",
                "ensure",
                "release-seq",
                "release-cctx",
                "validate",
                "lock",
                "unlock",
                "lock",
                "signal",
                "unlock",
            ]
        );
        assert_eq!(state.c_size, expected_error);
        assert_eq!(state.consumed_size, 11);
    }

    #[test]
    fn serial_turn_runs_ldm_before_checksum_and_advances_once() {
        let events = Rc::new(RefCell::new(Vec::new()));
        let wait_events = Rc::clone(&events);
        let ldm_window_events = Rc::clone(&events);
        let ldm_sequences_events = Rc::clone(&events);
        let ldm_publish_events = Rc::clone(&events);
        let checksum_events = Rc::clone(&events);
        let advance_events = Rc::clone(&events);
        let mut seq_store = ZstdMtRawSeqStore::default();

        serial_state_gen_sequences_with(
            &mut seq_store,
            ptr::null(),
            8,
            4,
            true,
            true,
            move |job_id| {
                assert_eq!(job_id, 4);
                wait_events.borrow_mut().push("wait");
                true
            },
            move |_seq_store, _src, src_size| {
                assert_eq!(src_size, 8);
                ldm_window_events.borrow_mut().push("ldm-window");
            },
            move |_seq_store, _src, src_size| {
                assert_eq!(src_size, 8);
                ldm_sequences_events.borrow_mut().push("ldm-sequences");
            },
            move || {
                ldm_publish_events.borrow_mut().push("ldm-publish");
            },
            move |_src, src_size| {
                assert_eq!(src_size, 8);
                checksum_events.borrow_mut().push("checksum");
            },
            move || advance_events.borrow_mut().push("advance"),
        );

        assert_eq!(
            &*events.borrow(),
            &[
                "wait",
                "ldm-window",
                "ldm-sequences",
                "ldm-publish",
                "checksum",
                "advance",
            ]
        );
    }

    #[test]
    fn serial_turn_skips_failed_predecessor_and_keeps_empty_ldm_turn() {
        let events = Rc::new(RefCell::new(Vec::new()));
        let wait_events = Rc::clone(&events);
        let advance_events = Rc::clone(&events);
        let mut seq_store = ZstdMtRawSeqStore::default();

        serial_state_gen_sequences_with(
            &mut seq_store,
            ptr::null(),
            0,
            9,
            true,
            true,
            move |_job_id| {
                wait_events.borrow_mut().push("wait");
                false
            },
            |_seq_store, _src, _src_size| panic!("skipped jobs must not update the LDM window"),
            |_seq_store, _src, _src_size| {
                panic!("skipped jobs must not generate LDM sequences")
            },
            || panic!("skipped jobs must not publish the LDM window"),
            |_src, _src_size| panic!("skipped jobs must not update the checksum"),
            move || advance_events.borrow_mut().push("advance"),
        );
        assert_eq!(&*events.borrow(), &["wait", "advance"]);

        let empty_events = Rc::new(RefCell::new(Vec::new()));
        let empty_ldm_window_events = Rc::clone(&empty_events);
        let empty_ldm_sequences_events = Rc::clone(&empty_events);
        let empty_ldm_publish_events = Rc::clone(&empty_events);
        let empty_advance_events = Rc::clone(&empty_events);
        serial_state_gen_sequences_with(
            &mut seq_store,
            ptr::null(),
            0,
            10,
            true,
            true,
            |_job_id| true,
            move |_seq_store, _src, src_size| {
                assert_eq!(src_size, 0);
                empty_ldm_window_events.borrow_mut().push("ldm-window");
            },
            move |_seq_store, _src, src_size| {
                assert_eq!(src_size, 0);
                empty_ldm_sequences_events
                    .borrow_mut()
                    .push("ldm-sequences");
            },
            move || {
                empty_ldm_publish_events.borrow_mut().push("ldm-publish");
            },
            |_src, _src_size| panic!("empty input must not update the checksum"),
            move || empty_advance_events.borrow_mut().push("advance"),
        );
        assert_eq!(
            &*empty_events.borrow(),
            &["ldm-window", "ldm-sequences", "ldm-publish", "advance"]
        );
    }

    #[test]
    fn serial_state_ensure_finished_owns_skip_and_wrapping_policy() {
        assert_eq!(
            ZSTDMT_rust_serialStateEnsureFinished(4, 4),
            ZSTDMT_serialStateEnsureFinishedResult {
                skip: 1,
                nextJobID: 5,
            }
        );
        assert_eq!(
            ZSTDMT_rust_serialStateEnsureFinished(3, 9),
            ZSTDMT_serialStateEnsureFinishedResult {
                skip: 1,
                nextJobID: 10,
            }
        );
        assert_eq!(
            ZSTDMT_rust_serialStateEnsureFinished(11, 9),
            ZSTDMT_serialStateEnsureFinishedResult {
                skip: 0,
                nextJobID: 11,
            }
        );
        assert_eq!(
            ZSTDMT_rust_serialStateEnsureFinished(c_uint::MAX, c_uint::MAX),
            ZSTDMT_serialStateEnsureFinishedResult {
                skip: 1,
                nextJobID: 0,
            }
        );
    }

    struct WaitForAllJobsTestContext {
        events: Vec<(c_uint, c_uint)>,
    }

    unsafe extern "C" fn wait_for_all_jobs_test_callback(
        context: *mut c_void,
        job_id: c_uint,
        done_job_id: c_uint,
    ) {
        unsafe {
            (*context.cast::<WaitForAllJobsTestContext>())
                .events
                .push((job_id, done_job_id));
        }
    }

    #[test]
    fn wait_for_all_jobs_uses_ring_order_and_advances_to_next_job() {
        let mut context = WaitForAllJobsTestContext { events: Vec::new() };

        let done_job_id = unsafe {
            ZSTDMT_rust_waitForAllJobsCompleted(
                3,
                6,
                3,
                (&mut context as *mut WaitForAllJobsTestContext).cast(),
                Some(wait_for_all_jobs_test_callback),
            )
        };

        assert_eq!(done_job_id, 6);
        assert_eq!(context.events, vec![(3, 3), (0, 4), (1, 5)]);
    }

    struct ReleaseAllJobsTestContext {
        job_ids: Vec<c_uint>,
    }

    unsafe extern "C" fn release_all_jobs_test_callback(context: *mut c_void, job_id: c_uint) {
        unsafe {
            (*context.cast::<ReleaseAllJobsTestContext>())
                .job_ids
                .push(job_id);
        }
    }

    #[test]
    fn release_all_jobs_visits_each_slot_in_order() {
        let mut context = ReleaseAllJobsTestContext {
            job_ids: Vec::new(),
        };

        unsafe {
            ZSTDMT_rust_releaseAllJobResources(
                3,
                (&mut context as *mut ReleaseAllJobsTestContext).cast(),
                Some(release_all_jobs_test_callback),
            )
        };

        assert_eq!(context.job_ids, vec![0, 1, 2, 3]);
    }

    #[test]
    fn compression_job_stops_on_non_first_chunk_error_and_cleans_up() {
        let state = Rc::new(RefCell::new(MockCompressionJob::default()));
        let acquire_state = Rc::clone(&state);
        let prepare_state = Rc::clone(&state);
        let sequence_state = Rc::clone(&state);
        let begin_state = Rc::clone(&state);
        let apply_state = Rc::clone(&state);
        let header_state = Rc::clone(&state);
        let compress_state = Rc::clone(&state);
        let expected_error = ERROR(ZstdErrorCode::DstSizeTooSmall);
        let finish = compression_job_finish_test_projection(&state, 23);

        compression_job_with(
            ZSTDMT_compressionJobProjection {
                firstJob: 0,
                lastJob: 1,
                frameHeaderState: ptr::null(),
                sequenceState: ptr::null(),
            },
            finish,
            move || {
                record_compression_job_event(&acquire_state, "acquire");
                0
            },
            move || record_compression_job_event(&prepare_state, "prepare"),
            move || record_compression_job_event(&sequence_state, "sequences"),
            move || {
                record_compression_job_event(&begin_state, "begin");
                0
            },
            move || record_compression_job_event(&apply_state, "apply"),
            move || {
                record_compression_job_event(&header_state, "header");
                0
            },
            move |last_job| {
                assert_eq!(last_job, 1);
                record_compression_job_event(&compress_state, "compress");
                ZSTDMT_chunkProcessResult {
                    error: expected_error,
                    lastBlockSize: 9,
                }
            },
            || panic!("a failed chunk must not be traced"),
        );

        let state = state.borrow();
        assert_eq!(
            state.events,
            vec![
                "acquire",
                "prepare",
                "sequences",
                "begin",
                "apply",
                "header",
                "compress",
                "lock",
                "unlock",
                "ensure",
                "release-seq",
                "release-cctx",
                "validate",
                "lock",
                "unlock",
                "lock",
                "signal",
                "unlock",
            ]
        );
        assert_eq!(state.c_size, expected_error);
        assert_eq!(state.consumed_size, 23);
    }

    fn init_projection() -> ZSTDMT_initCStreamProjection {
        ZSTDMT_initCStreamProjection {
            requestedNbWorkers: 4,
            currentNbWorkers: 2,
            jobSize: 1,
            jobSizeMin: DEFAULT_ZSTDMT_JOBSIZE_MIN,
            jobSizeMax: DEFAULT_ZSTDMT_JOBSIZE_MAX,
            enableLdm: 0,
            windowLog: 20,
            chainLog: 16,
            strategy: ZSTD_FAST,
            overlapLog: 1,
            rsyncable: 1,
            roundBuffCapacity: 0,
            allJobsCompleted: 0,
        }
    }

    #[derive(Default)]
    struct ResetStreamTestContext {
        events: Vec<&'static str>,
    }

    unsafe extern "C" fn reset_stream_test_round_buffer(context: *mut c_void) {
        unsafe {
            (*context.cast::<ResetStreamTestContext>())
                .events
                .push("round");
        }
    }

    unsafe extern "C" fn reset_stream_test_input_buffer(context: *mut c_void) {
        unsafe {
            (*context.cast::<ResetStreamTestContext>())
                .events
                .push("input");
        }
    }

    unsafe extern "C" fn reset_stream_test_job_ids(context: *mut c_void) {
        unsafe {
            (*context.cast::<ResetStreamTestContext>())
                .events
                .push("jobs");
        }
    }

    unsafe extern "C" fn reset_stream_test_frame_flags(context: *mut c_void) {
        unsafe {
            (*context.cast::<ResetStreamTestContext>())
                .events
                .push("flags");
        }
    }

    unsafe extern "C" fn reset_stream_test_progress(context: *mut c_void) {
        unsafe {
            (*context.cast::<ResetStreamTestContext>())
                .events
                .push("progress");
        }
    }

    fn reset_stream_test_state(context: *mut c_void) -> ZSTDMT_RustResetStreamState {
        ZSTDMT_RustResetStreamState {
            callback_context: context,
            reset_round_buffer: Some(reset_stream_test_round_buffer),
            reset_input_buffer: Some(reset_stream_test_input_buffer),
            reset_job_ids: Some(reset_stream_test_job_ids),
            reset_frame_flags: Some(reset_stream_test_frame_flags),
            reset_progress: Some(reset_stream_test_progress),
        }
    }

    #[test]
    fn reset_stream_preserves_callback_order() {
        let mut context = ResetStreamTestContext::default();
        let state = reset_stream_test_state((&mut context as *mut ResetStreamTestContext).cast());

        unsafe { ZSTDMT_rust_resetStream(&state) };

        assert_eq!(
            context.events,
            vec!["round", "input", "jobs", "flags", "progress"]
        );
    }

    #[test]
    fn reset_stream_ignores_null_or_incomplete_state() {
        let mut context = ResetStreamTestContext::default();
        let state = ZSTDMT_RustResetStreamState {
            callback_context: (&mut context as *mut ResetStreamTestContext).cast(),
            reset_round_buffer: None,
            reset_input_buffer: Some(reset_stream_test_input_buffer),
            reset_job_ids: Some(reset_stream_test_job_ids),
            reset_frame_flags: Some(reset_stream_test_frame_flags),
            reset_progress: Some(reset_stream_test_progress),
        };

        unsafe {
            ZSTDMT_rust_resetStream(&state);
            ZSTDMT_rust_resetStream(ptr::null());
        }

        assert!(context.events.is_empty());
    }

    #[derive(Default)]
    struct SerialResetTestContext {
        events: Vec<&'static str>,
        ldm_params: ZSTDMT_serialResetLdmParameters,
        max_nb_seq: usize,
        resize_result: c_int,
        hash_size: usize,
        num_buckets: usize,
        bucket_log: c_uint,
        previous_bucket_log: c_uint,
        published_job_size: usize,
        published_job_size_u32: u32,
    }

    unsafe extern "C" fn serial_reset_test_set_ldm_params(
        context: *mut c_void,
        enable_ldm: c_int,
        hash_log: c_uint,
        bucket_size_log: c_uint,
        min_match_length: c_uint,
        hash_rate_log: c_uint,
        window_log: c_uint,
    ) {
        unsafe {
            let context = &mut *context.cast::<SerialResetTestContext>();
            context.events.push("ldm-params");
            context.ldm_params = ZSTDMT_serialResetLdmParameters {
                enableLdm: enable_ldm,
                hashLog: hash_log,
                bucketSizeLog: bucket_size_log,
                minMatchLength: min_match_length,
                hashRateLog: hash_rate_log,
                windowLog: window_log,
            };
        }
    }

    unsafe extern "C" fn serial_reset_test_next_job(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialResetTestContext>())
                .events
                .push("next-job");
        }
    }

    unsafe extern "C" fn serial_reset_test_checksum(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialResetTestContext>())
                .events
                .push("checksum");
        }
    }

    unsafe extern "C" fn serial_reset_test_set_nb_seq(context: *mut c_void, max_nb_seq: usize) {
        unsafe {
            let context = &mut *context.cast::<SerialResetTestContext>();
            context.events.push("seq-size");
            context.max_nb_seq = max_nb_seq;
        }
    }

    unsafe extern "C" fn serial_reset_test_window(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialResetTestContext>())
                .events
                .push("window");
        }
    }

    unsafe extern "C" fn serial_reset_test_resize(
        context: *mut c_void,
        hash_size: usize,
        num_buckets: usize,
        bucket_log: c_uint,
        previous_bucket_log: c_uint,
    ) -> c_int {
        unsafe {
            let context = &mut *context.cast::<SerialResetTestContext>();
            context.events.push("resize");
            context.hash_size = hash_size;
            context.num_buckets = num_buckets;
            context.bucket_log = bucket_log;
            context.previous_bucket_log = previous_bucket_log;
            context.resize_result
        }
    }

    unsafe extern "C" fn serial_reset_test_zero(
        context: *mut c_void,
        _hash_size: usize,
        _num_buckets: usize,
    ) {
        unsafe {
            (*context.cast::<SerialResetTestContext>())
                .events
                .push("zero");
        }
    }

    unsafe extern "C" fn serial_reset_test_dictionary(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialResetTestContext>())
                .events
                .push("dictionary");
        }
    }

    unsafe extern "C" fn serial_reset_test_copy_window(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialResetTestContext>())
                .events
                .push("copy-window");
        }
    }

    unsafe extern "C" fn serial_reset_test_publish(context: *mut c_void, job_size: usize) {
        unsafe {
            let context = &mut *context.cast::<SerialResetTestContext>();
            context.events.push("publish");
            context.published_job_size = job_size;
            context.published_job_size_u32 = job_size as u32;
        }
    }

    type SerialResetTestCallbacks = (
        Option<ZSTDMT_serialResetSetLdmParamsFn>,
        Option<ZSTDMT_serialResetVoidFn>,
        Option<ZSTDMT_serialResetVoidFn>,
        Option<ZSTDMT_serialResetSetNbSeqFn>,
        Option<ZSTDMT_serialResetVoidFn>,
        Option<ZSTDMT_serialResetResizeFn>,
        Option<ZSTDMT_serialResetZeroTablesFn>,
        Option<ZSTDMT_serialResetVoidFn>,
        Option<ZSTDMT_serialResetVoidFn>,
        Option<ZSTDMT_serialResetPublishFn>,
    );

    fn serial_reset_test_callbacks() -> SerialResetTestCallbacks {
        (
            Some(serial_reset_test_set_ldm_params),
            Some(serial_reset_test_next_job),
            Some(serial_reset_test_checksum),
            Some(serial_reset_test_set_nb_seq),
            Some(serial_reset_test_window),
            Some(serial_reset_test_resize),
            Some(serial_reset_test_zero),
            Some(serial_reset_test_dictionary),
            Some(serial_reset_test_copy_window),
            Some(serial_reset_test_publish),
        )
    }

    fn serial_reset_test_projection(
        enable_ldm: c_int,
        checksum_enabled: c_int,
        job_size: usize,
    ) -> ZSTDMT_RustSerialResetProjection {
        ZSTDMT_RustSerialResetProjection {
            enableLdm: enable_ldm,
            checksumEnabled: checksum_enabled,
            jobSize: job_size,
            windowLog: 20,
            strategy: ZSTD_GREEDY,
            hashLog: 0,
            bucketSizeLog: 0,
            minMatchLength: 0,
            hashRateLog: 0,
            previousHashLog: 10,
            previousBucketSizeLog: 4,
            ldmEntrySize: 8,
        }
    }

    #[test]
    fn serial_state_reset_enabled_prepares_scalars_and_preserves_order() {
        let mut context = SerialResetTestContext::default();
        let callbacks = serial_reset_test_callbacks();
        let projection = serial_reset_test_projection(ZSTD_PS_ENABLE, 1, 640);

        let result = unsafe {
            ZSTDMT_rust_serialStateReset(
                &projection,
                (&mut context as *mut SerialResetTestContext).cast(),
                callbacks.0,
                callbacks.1,
                callbacks.2,
                callbacks.3,
                callbacks.4,
                callbacks.5,
                callbacks.6,
                callbacks.7,
                callbacks.8,
                callbacks.9,
            )
        };

        assert_eq!(result, 0);
        assert_eq!(
            context.ldm_params,
            ZSTDMT_serialResetLdmParameters {
                enableLdm: ZSTD_PS_ENABLE,
                hashLog: 14,
                bucketSizeLog: 4,
                minMatchLength: 64,
                hashRateLog: 6,
                windowLog: 20,
            }
        );
        assert_eq!(context.max_nb_seq, 10);
        assert_eq!(context.hash_size, 1 << 14 << 3);
        assert_eq!(context.num_buckets, 1 << 10);
        assert_eq!(context.bucket_log, 10);
        assert_eq!(context.previous_bucket_log, 6);
        assert_eq!(context.published_job_size, 640);
        assert_eq!(
            context.events,
            vec![
                "ldm-params",
                "next-job",
                "checksum",
                "seq-size",
                "window",
                "resize",
                "zero",
                "dictionary",
                "copy-window",
                "publish",
            ]
        );
    }

    #[test]
    fn serial_state_reset_disabled_zeroes_ldm_scalars_and_skips_ldm_work() {
        let mut context = SerialResetTestContext::default();
        let callbacks = serial_reset_test_callbacks();
        let mut projection = serial_reset_test_projection(ZSTD_PS_DISABLE, 0, 640);
        projection.hashLog = 20;
        projection.bucketSizeLog = 4;
        projection.minMatchLength = 32;
        projection.hashRateLog = 5;

        let result = unsafe {
            ZSTDMT_rust_serialStateReset(
                &projection,
                (&mut context as *mut SerialResetTestContext).cast(),
                callbacks.0,
                callbacks.1,
                callbacks.2,
                callbacks.3,
                callbacks.4,
                callbacks.5,
                callbacks.6,
                callbacks.7,
                callbacks.8,
                callbacks.9,
            )
        };

        assert_eq!(result, 0);
        assert_eq!(
            context.ldm_params,
            ZSTDMT_serialResetLdmParameters::default()
        );
        assert_eq!(context.max_nb_seq, 0);
        assert_eq!(context.published_job_size, 640);
        assert_eq!(context.events, vec!["ldm-params", "next-job", "publish"]);
    }

    #[test]
    fn serial_state_reset_preserves_size_t_job_size_overflow_and_cast() {
        let mut context = SerialResetTestContext::default();
        let callbacks = serial_reset_test_callbacks();
        let projection = serial_reset_test_projection(ZSTD_PS_ENABLE, 0, usize::MAX);

        let result = unsafe {
            ZSTDMT_rust_serialStateReset(
                &projection,
                (&mut context as *mut SerialResetTestContext).cast(),
                callbacks.0,
                callbacks.1,
                callbacks.2,
                callbacks.3,
                callbacks.4,
                callbacks.5,
                callbacks.6,
                callbacks.7,
                callbacks.8,
                callbacks.9,
            )
        };

        assert_eq!(result, 0);
        assert_eq!(context.max_nb_seq, usize::MAX / 64);
        assert_eq!(context.published_job_size, usize::MAX);
        assert_eq!(context.published_job_size_u32, usize::MAX as u32);
    }

    #[test]
    fn serial_state_reset_stops_before_zeroing_after_resize_failure() {
        let mut context = SerialResetTestContext {
            resize_result: 1,
            ..Default::default()
        };
        let callbacks = serial_reset_test_callbacks();
        let projection = serial_reset_test_projection(ZSTD_PS_ENABLE, 0, 640);

        let result = unsafe {
            ZSTDMT_rust_serialStateReset(
                &projection,
                (&mut context as *mut SerialResetTestContext).cast(),
                callbacks.0,
                callbacks.1,
                callbacks.2,
                callbacks.3,
                callbacks.4,
                callbacks.5,
                callbacks.6,
                callbacks.7,
                callbacks.8,
                callbacks.9,
            )
        };

        assert_eq!(result, 1);
        assert_eq!(
            context.events,
            vec!["ldm-params", "next-job", "seq-size", "window", "resize"]
        );
        assert_eq!(
            ZSTDMT_rust_initSerialResetResult(result as c_int),
            ERROR(ZstdErrorCode::MemoryAllocation)
        );
    }

    #[test]
    fn serial_dictionary_policy_resets_then_loads_only_raw_content() {
        let events = RefCell::new(Vec::<(&'static str, usize)>::new());
        serial_state_load_dictionary_with(
            ZSTDMT_RustSerialDictionaryProjection {
                dict: ptr::null(),
                dictSize: 12,
                dictContentType: ZSTD_DCT_RAW_CONTENT,
                forceWindow: 0,
            },
            |dict, dict_size| {
                assert!(dict.is_null());
                events.borrow_mut().push(("load", dict_size));
                23
            },
            |loaded_dict_end| events.borrow_mut().push(("end", loaded_dict_end as usize)),
        );
        assert_eq!(
            events.into_inner(),
            vec![("end", 0), ("load", 12), ("end", 23)]
        );

        let events = RefCell::new(Vec::<(&'static str, usize)>::new());
        serial_state_load_dictionary_with(
            ZSTDMT_RustSerialDictionaryProjection {
                dict: ptr::null(),
                dictSize: 12,
                dictContentType: ZSTD_DCT_RAW_CONTENT,
                forceWindow: 1,
            },
            |_, dict_size| {
                events.borrow_mut().push(("load", dict_size));
                23
            },
            |loaded_dict_end| events.borrow_mut().push(("end", loaded_dict_end as usize)),
        );
        assert_eq!(events.into_inner(), vec![("end", 0), ("load", 12)]);

        let events = RefCell::new(Vec::<(&'static str, usize)>::new());
        serial_state_load_dictionary_with(
            ZSTDMT_RustSerialDictionaryProjection {
                dict: ptr::null(),
                dictSize: 12,
                dictContentType: 2,
                forceWindow: 0,
            },
            |_, _| panic!("non-raw dictionary must not be loaded"),
            |loaded_dict_end| events.borrow_mut().push(("end", loaded_dict_end as usize)),
        );
        assert_eq!(events.into_inner(), vec![("end", 0)]);

        let events = RefCell::new(Vec::<(&'static str, usize)>::new());
        serial_state_load_dictionary_with(
            ZSTDMT_RustSerialDictionaryProjection {
                dict: ptr::null(),
                dictSize: 0,
                dictContentType: ZSTD_DCT_RAW_CONTENT,
                forceWindow: 0,
            },
            |_, _| panic!("empty dictionary must not be loaded"),
            |loaded_dict_end| events.borrow_mut().push(("end", loaded_dict_end as usize)),
        );
        assert_eq!(events.into_inner(), vec![("end", 0)]);
    }

    #[derive(Default)]
    struct SerialLifecycleTestContext {
        events: Vec<&'static str>,
        init_errors: [c_int; 4],
    }

    unsafe fn serial_lifecycle_init_step(
        context: *mut c_void,
        index: usize,
        event: &'static str,
    ) -> c_int {
        let context = unsafe { &mut *context.cast::<SerialLifecycleTestContext>() };
        context.events.push(event);
        context.init_errors[index]
    }

    unsafe extern "C" fn serial_lifecycle_zero(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialLifecycleTestContext>())
                .events
                .push("zero");
        }
    }

    unsafe extern "C" fn serial_lifecycle_init_mutex(context: *mut c_void) -> c_int {
        unsafe { serial_lifecycle_init_step(context, 0, "mutex") }
    }

    unsafe extern "C" fn serial_lifecycle_init_cond(context: *mut c_void) -> c_int {
        unsafe { serial_lifecycle_init_step(context, 1, "cond") }
    }

    unsafe extern "C" fn serial_lifecycle_init_ldm_mutex(context: *mut c_void) -> c_int {
        unsafe { serial_lifecycle_init_step(context, 2, "ldm-mutex") }
    }

    unsafe extern "C" fn serial_lifecycle_init_ldm_cond(context: *mut c_void) -> c_int {
        unsafe { serial_lifecycle_init_step(context, 3, "ldm-cond") }
    }

    unsafe extern "C" fn serial_lifecycle_destroy_mutex(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialLifecycleTestContext>())
                .events
                .push("destroy-mutex");
        }
    }

    unsafe extern "C" fn serial_lifecycle_destroy_cond(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialLifecycleTestContext>())
                .events
                .push("destroy-cond");
        }
    }

    unsafe extern "C" fn serial_lifecycle_destroy_ldm_mutex(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialLifecycleTestContext>())
                .events
                .push("destroy-ldm-mutex");
        }
    }

    unsafe extern "C" fn serial_lifecycle_destroy_ldm_cond(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialLifecycleTestContext>())
                .events
                .push("destroy-ldm-cond");
        }
    }

    unsafe extern "C" fn serial_lifecycle_free_tables(context: *mut c_void) {
        unsafe {
            (*context.cast::<SerialLifecycleTestContext>())
                .events
                .push("free-tables");
        }
    }

    #[test]
    fn serial_state_lifecycle_preserves_order_and_aggregates_init_errors() {
        let mut context = SerialLifecycleTestContext {
            init_errors: [0, 2, 0, 4],
            ..Default::default()
        };
        let opaque = (&mut context as *mut SerialLifecycleTestContext).cast();

        let result = unsafe {
            ZSTDMT_rust_serialStateInit(
                opaque,
                Some(serial_lifecycle_zero),
                Some(serial_lifecycle_init_mutex),
                Some(serial_lifecycle_init_cond),
                Some(serial_lifecycle_init_ldm_mutex),
                Some(serial_lifecycle_init_ldm_cond),
            )
        };
        assert_eq!(result, 6);

        unsafe {
            ZSTDMT_rust_serialStateFree(
                opaque,
                Some(serial_lifecycle_destroy_mutex),
                Some(serial_lifecycle_destroy_cond),
                Some(serial_lifecycle_destroy_ldm_mutex),
                Some(serial_lifecycle_destroy_ldm_cond),
                Some(serial_lifecycle_free_tables),
            )
        };

        assert_eq!(
            context.events,
            vec![
                "zero",
                "mutex",
                "cond",
                "ldm-mutex",
                "ldm-cond",
                "destroy-mutex",
                "destroy-cond",
                "destroy-ldm-mutex",
                "destroy-ldm-cond",
                "free-tables",
            ]
        );
    }

    #[test]
    fn init_serial_reset_result_maps_nonzero_status_to_memory_error() {
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);

        assert_eq!(init_serial_reset_result(0), 0);
        assert_eq!(ZSTDMT_rust_initSerialResetResult(0), 0);
        for status in [1, -1, 2] {
            assert_eq!(init_serial_reset_result(status), expected_error);
            assert_eq!(ZSTDMT_rust_initSerialResetResult(status), expected_error);
        }
    }

    fn mt_cparams_update_projection() -> ZSTDMT_CParamsUpdateProjection {
        ZSTDMT_CParamsUpdateProjection {
            compressionLevel: 3,
            cctxSrcSizeHint: 0,
            srcSizeHint: u64::MAX,
            dictSize: 0,
            mode: 0,
            enableLdm: ZSTD_PS_DISABLE,
            ldmDefaultWindowLog: 27,
            overrides: ZSTD_compressionParameters::default(),
            useRowMatchFinder: 0,
            exclusionMask: 0,
            savedWindowLog: 18,
        }
    }

    fn expected_mt_cparams_update(
        projection: ZSTDMT_CParamsUpdateProjection,
    ) -> ZSTD_compressionParameters {
        ZSTD_rust_params_updateCParamsWhileCompressing(
            ZSTD_rust_params_updateCParamsWhileCompressingState {
                compressionLevel: projection.compressionLevel,
                cctxSrcSizeHint: projection.cctxSrcSizeHint,
                srcSizeHint: projection.srcSizeHint,
                dictSize: projection.dictSize,
                mode: projection.mode,
                enableLdm: projection.enableLdm,
                ldmDefaultWindowLog: projection.ldmDefaultWindowLog,
                overrides: projection.overrides,
                useRowMatchFinder: projection.useRowMatchFinder,
                exclusionMask: projection.exclusionMask,
                savedWindowLog: projection.savedWindowLog,
            },
        )
    }

    #[test]
    fn mt_update_parameters_carries_level_with_next_job_params() {
        let projection = mt_cparams_update_projection();
        let expected = expected_mt_cparams_update(projection);
        let result = update_cparams_while_compressing(projection);

        assert_eq!(result.compressionLevel, projection.compressionLevel);
        assert_eq!(result.cParams, expected);
        assert_eq!(result.cParams.windowLog, projection.savedWindowLog);
        assert_eq!(
            ZSTDMT_rust_updateCParamsWhileCompressing(projection),
            result
        );
    }

    #[test]
    fn mt_update_parameters_uses_context_size_hint_only_for_unknown_size() {
        let mut from_context_hint = mt_cparams_update_projection();
        from_context_hint.cctxSrcSizeHint = 1 << 20;

        let mut explicit_size = mt_cparams_update_projection();
        explicit_size.srcSizeHint = 1 << 20;

        assert_eq!(
            update_cparams_while_compressing(from_context_hint).cParams,
            update_cparams_while_compressing(explicit_size).cParams
        );

        let mut explicit_with_stale_hint = explicit_size;
        explicit_with_stale_hint.cctxSrcSizeHint = 1;
        assert_eq!(
            update_cparams_while_compressing(explicit_size).cParams,
            update_cparams_while_compressing(explicit_with_stale_hint).cParams
        );
    }

    #[test]
    fn mt_update_parameters_restores_active_window_after_ldm_and_overrides() {
        let mut projection = mt_cparams_update_projection();
        projection.compressionLevel = -1;
        projection.enableLdm = ZSTD_PS_ENABLE;
        projection.ldmDefaultWindowLog = 27;
        projection.overrides.strategy = ZSTD_BTULTRA2;
        projection.savedWindowLog = 19;

        let result = update_cparams_while_compressing(projection);

        assert_eq!(result.compressionLevel, -1);
        assert_eq!(result.cParams.strategy, ZSTD_BTULTRA2);
        assert_eq!(result.cParams.windowLog, projection.savedWindowLog);
        assert_ne!(result.cParams.windowLog, projection.ldmDefaultWindowLog);
    }

    #[test]
    fn dictionary_policy_classifies_copy_borrow_and_raw_prefix_branches() {
        assert_eq!(
            classify_prepare_dictionary(ZSTDMT_RustDictionaryProjection {
                hasDictionary: 1,
                rawContent: 0,
            }),
            PrepareDictionaryAction::CreateCopied
        );
        assert_eq!(
            classify_prepare_dictionary(ZSTDMT_RustDictionaryProjection {
                hasDictionary: 0,
                rawContent: 1,
            }),
            PrepareDictionaryAction::AttachBorrowed
        );

        for (projection, expected) in [
            (
                ZSTDMT_RustDictionaryProjection {
                    hasDictionary: 0,
                    rawContent: 0,
                },
                UpdateDictionaryAction::AttachBorrowed,
            ),
            (
                ZSTDMT_RustDictionaryProjection {
                    hasDictionary: 1,
                    rawContent: 1,
                },
                UpdateDictionaryAction::SetRawPrefix,
            ),
            (
                ZSTDMT_RustDictionaryProjection {
                    hasDictionary: 1,
                    rawContent: 0,
                },
                UpdateDictionaryAction::CreateReferenced,
            ),
        ] {
            assert_eq!(classify_update_dictionary(projection), expected);
        }
    }

    #[test]
    fn prepare_dictionary_releases_before_copy_or_borrow_action() {
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
        let events = RefCell::new(Vec::new());
        let result = prepare_c_stream_dictionary_with(
            ZSTDMT_RustDictionaryProjection {
                hasDictionary: 1,
                rawContent: 0,
            },
            || events.borrow_mut().push("release"),
            || {
                events.borrow_mut().push("copy");
                expected_error
            },
            || panic!("copy branch must not attach borrowed dictionary"),
        );
        assert_eq!(result, expected_error);
        assert_eq!(events.into_inner(), vec!["release", "copy"]);

        let events = RefCell::new(Vec::new());
        let result = prepare_c_stream_dictionary_with(
            ZSTDMT_RustDictionaryProjection {
                hasDictionary: 0,
                rawContent: 0,
            },
            || events.borrow_mut().push("release"),
            || panic!("borrow branch must not create a copied dictionary"),
            || events.borrow_mut().push("borrow"),
        );
        assert_eq!(result, 0);
        assert_eq!(events.into_inner(), vec!["release", "borrow"]);
    }

    #[test]
    fn update_dictionary_clears_before_borrow_raw_or_reference_action() {
        let events = RefCell::new(Vec::new());
        let result = update_c_stream_dictionary_with(
            ZSTDMT_RustDictionaryProjection {
                hasDictionary: 0,
                rawContent: 0,
            },
            || events.borrow_mut().push("clear"),
            || events.borrow_mut().push("borrow"),
            || panic!("borrow branch must not set a raw prefix"),
            || panic!("borrow branch must not create a referenced dictionary"),
        );
        assert_eq!(result, 0);
        assert_eq!(events.into_inner(), vec!["clear", "borrow"]);

        let events = RefCell::new(Vec::new());
        let result = update_c_stream_dictionary_with(
            ZSTDMT_RustDictionaryProjection {
                hasDictionary: 1,
                rawContent: 1,
            },
            || events.borrow_mut().push("clear"),
            || panic!("raw-prefix branch must not attach a borrowed CDict"),
            || events.borrow_mut().push("raw"),
            || panic!("raw-prefix branch must not create a referenced dictionary"),
        );
        assert_eq!(result, 0);
        assert_eq!(events.into_inner(), vec!["clear", "raw"]);

        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
        let events = RefCell::new(Vec::new());
        let result = update_c_stream_dictionary_with(
            ZSTDMT_RustDictionaryProjection {
                hasDictionary: 1,
                rawContent: 0,
            },
            || events.borrow_mut().push("clear"),
            || panic!("reference branch must not attach a borrowed CDict"),
            || panic!("reference branch must not set a raw prefix"),
            || {
                events.borrow_mut().push("reference");
                expected_error
            },
        );
        assert_eq!(result, expected_error);
        assert_eq!(events.into_inner(), vec!["clear", "reference"]);
    }

    #[test]
    fn init_c_stream_preserves_success_order_and_normalization() {
        let projection = init_projection();
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let values = Rc::new(RefCell::new(Vec::<(&'static str, usize)>::new()));
        let resize_events = Rc::clone(&events);
        let wait_events = Rc::clone(&events);
        let release_events = Rc::clone(&events);
        let apply_events = Rc::clone(&events);
        let apply_values = Rc::clone(&values);
        let prepare_events = Rc::clone(&events);
        let prefix_events = Rc::clone(&events);
        let prefix_values = Rc::clone(&values);
        let section_events = Rc::clone(&events);
        let section_values = Rc::clone(&values);
        let rsync_events = Rc::clone(&events);
        let buffer_events = Rc::clone(&events);
        let buffer_values = Rc::clone(&values);
        let round_events = Rc::clone(&events);
        let round_values = Rc::clone(&values);
        let reset_events = Rc::clone(&events);
        let update_events = Rc::clone(&events);
        let serial_events = Rc::clone(&events);
        let serial_values = Rc::clone(&values);

        let result = init_c_stream_with(
            projection,
            move |workers| {
                assert_eq!(workers, 4);
                resize_events.borrow_mut().push("resize");
                0
            },
            move || wait_events.borrow_mut().push("wait"),
            move || release_events.borrow_mut().push("release"),
            move |job_size| {
                apply_events.borrow_mut().push("apply");
                apply_values.borrow_mut().push(("job", job_size));
            },
            move || {
                prepare_events.borrow_mut().push("prepare-dict");
                0
            },
            move |size| {
                prefix_events.borrow_mut().push("prefix");
                prefix_values.borrow_mut().push(("prefix", size));
            },
            move |size| {
                section_events.borrow_mut().push("section");
                section_values.borrow_mut().push(("section", size));
            },
            move |_hit_mask, _prime_power| rsync_events.borrow_mut().push("rsync"),
            move |size| {
                buffer_events.borrow_mut().push("buffer");
                buffer_values.borrow_mut().push(("buffer", size));
            },
            move |capacity| {
                round_events.borrow_mut().push("round");
                round_values.borrow_mut().push(("round", capacity));
                0
            },
            move || reset_events.borrow_mut().push("reset"),
            move || {
                update_events.borrow_mut().push("update-dict");
                0
            },
            move |size| {
                serial_events.borrow_mut().push("serial");
                serial_values.borrow_mut().push(("serial", size));
                0
            },
        );

        assert_eq!(result, 0);
        assert_eq!(
            events.borrow().as_slice(),
            &[
                "resize",
                "wait",
                "release",
                "apply",
                "prepare-dict",
                "prefix",
                "section",
                "rsync",
                "buffer",
                "round",
                "reset",
                "update-dict",
                "serial",
            ]
        );
        assert_eq!(values.borrow()[0], ("job", DEFAULT_ZSTDMT_JOBSIZE_MIN));
        assert_eq!(values.borrow()[1], ("prefix", 0));
        assert_eq!(values.borrow()[2], ("section", DEFAULT_ZSTDMT_JOBSIZE_MIN));
        assert_eq!(values.borrow()[5], ("serial", DEFAULT_ZSTDMT_JOBSIZE_MIN));
        assert!(values.borrow()[3].1 > 0);
        assert!(values.borrow()[4].1 > 0);
    }

    #[test]
    fn init_c_stream_stops_before_normalization_when_resize_fails() {
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
        let result = init_c_stream_with(
            init_projection(),
            |_| expected_error,
            || panic!("a failed resize must stop initialization"),
            || panic!("a failed resize must stop initialization"),
            |_| panic!("a failed resize must stop initialization"),
            || panic!("a failed resize must stop initialization"),
            |_| panic!("a failed resize must stop initialization"),
            |_| panic!("a failed resize must stop initialization"),
            |_, _| panic!("a failed resize must stop initialization"),
            |_| panic!("a failed resize must stop initialization"),
            |_| panic!("a failed resize must stop initialization"),
            || panic!("a failed resize must stop initialization"),
            || panic!("a failed resize must stop initialization"),
            |_| panic!("a failed resize must stop initialization"),
        );

        assert_eq!(result, expected_error);
    }

    #[test]
    fn init_c_stream_stops_after_round_buffer_error() {
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
        let mut projection = init_projection();
        projection.currentNbWorkers = projection.requestedNbWorkers;
        projection.allJobsCompleted = 1;
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let result = init_c_stream_with(
            projection,
            |_| panic!("workers already match"),
            || panic!("all jobs are complete"),
            || panic!("all jobs are complete"),
            {
                let events = Rc::clone(&events);
                move |_| events.borrow_mut().push("apply")
            },
            {
                let events = Rc::clone(&events);
                move || {
                    events.borrow_mut().push("prepare");
                    0
                }
            },
            {
                let events = Rc::clone(&events);
                move |_| events.borrow_mut().push("prefix")
            },
            {
                let events = Rc::clone(&events);
                move |_| events.borrow_mut().push("section")
            },
            {
                let events = Rc::clone(&events);
                move |_, _| events.borrow_mut().push("rsync")
            },
            {
                let events = Rc::clone(&events);
                move |_| events.borrow_mut().push("buffer")
            },
            {
                let events = Rc::clone(&events);
                move |_| {
                    events.borrow_mut().push("round");
                    expected_error
                }
            },
            {
                let events = Rc::clone(&events);
                move || events.borrow_mut().push("reset")
            },
            {
                let events = Rc::clone(&events);
                move || {
                    events.borrow_mut().push("update");
                    0
                }
            },
            |_| panic!("serial reset must follow successful dictionary update"),
        );

        assert_eq!(result, expected_error);
        assert_eq!(
            events.borrow().as_slice(),
            &["apply", "prepare", "prefix", "section", "rsync", "buffer", "round"]
        );
    }

    unsafe extern "C" fn mock_compress_continue(
        cctx: *mut c_void,
        _dst: *mut c_void,
        _dst_capacity: usize,
        _src: *const c_void,
        src_size: usize,
    ) -> usize {
        let compressor = unsafe { &mut *cctx.cast::<MockChunkCompressor>() };
        let call_number = compressor.continue_inputs.len();
        compressor.continue_inputs.push(src_size);
        compressor
            .continue_results
            .get(call_number)
            .copied()
            .unwrap_or(0)
    }

    unsafe extern "C" fn mock_compress_end(
        cctx: *mut c_void,
        _dst: *mut c_void,
        _dst_capacity: usize,
        _src: *const c_void,
        src_size: usize,
    ) -> usize {
        let compressor = unsafe { &mut *cctx.cast::<MockChunkCompressor>() };
        let call_number = compressor.end_inputs.len();
        compressor.end_inputs.push(src_size);
        compressor
            .end_results
            .get(call_number)
            .copied()
            .unwrap_or(0)
    }

    unsafe extern "C" fn mock_chunk_progress(context: *mut c_void, c_size: usize, consumed: usize) {
        let progress = unsafe { &mut *context.cast::<MockChunkProgress>() };
        progress.calls.push((c_size, consumed));
    }

    unsafe extern "C" fn mock_job_progress_lock(context: *mut c_void) {
        unsafe { (*context.cast::<MockJobProgress>()).events.push("lock") };
    }

    unsafe extern "C" fn mock_job_progress_signal(context: *mut c_void) {
        unsafe { (*context.cast::<MockJobProgress>()).events.push("signal") };
    }

    unsafe extern "C" fn mock_job_progress_unlock(context: *mut c_void) {
        unsafe { (*context.cast::<MockJobProgress>()).events.push("unlock") };
    }

    unsafe extern "C" fn mock_job_progress_debug(
        context: *mut c_void,
        c_size: usize,
        total_c_size: usize,
    ) {
        let progress = unsafe { &mut *context.cast::<MockJobProgress>() };
        progress.events.push("debug");
        progress.debug_calls.push((c_size, total_c_size));
    }

    fn run_mock_chunk_loop(
        compressor: &mut MockChunkCompressor,
        progress: &mut MockChunkProgress,
        src_size: usize,
        chunk_size: usize,
        last_job: c_uint,
    ) -> ZSTDMT_chunkProcessResult {
        let src = [0u8; 32];
        let mut dst = [0u8; 64];
        unsafe {
            compress_job_chunks_with(
                (compressor as *mut MockChunkCompressor).cast(),
                src.as_ptr().cast(),
                src_size,
                dst.as_mut_ptr().cast(),
                dst.len(),
                chunk_size,
                last_job,
                (progress as *mut MockChunkProgress).cast(),
                Some(mock_chunk_progress),
                mock_compress_continue,
                mock_compress_end,
            )
        }
    }

    #[test]
    fn compression_job_progress_preserves_publication_order_and_wraps_c_size() {
        let mut c_size = usize::MAX - 1;
        let mut consumed = 0;
        let mut progress = MockJobProgress::default();
        let projection = ZSTDMT_compressionJobProgressProjection {
            callbackContext: (&mut progress as *mut MockJobProgress).cast(),
            cSize: &mut c_size,
            consumed: &mut consumed,
            lock: Some(mock_job_progress_lock),
            signal: Some(mock_job_progress_signal),
            unlock: Some(mock_job_progress_unlock),
            debug: Some(mock_job_progress_debug),
        };

        unsafe {
            ZSTDMT_rust_compressionJobProgress(
                (&projection as *const ZSTDMT_compressionJobProgressProjection)
                    .cast_mut()
                    .cast(),
                3,
                17,
            )
        };

        assert_eq!(c_size, 1);
        assert_eq!(consumed, 17);
        assert_eq!(progress.events, ["lock", "debug", "signal", "unlock"]);
        assert_eq!(progress.debug_calls, [(3, 1)]);
    }

    fn run_output_publication(
        output: &mut [u8],
        output_size: usize,
        output_pos: usize,
        job: &[u8],
        job_capacity: usize,
        c_size: usize,
        dst_flushed: usize,
    ) -> ZSTDMT_flushPublicationResult {
        unsafe {
            publish_job_output_with(
                output.as_mut_ptr().cast(),
                output_size,
                output_pos,
                job.as_ptr().cast(),
                job_capacity,
                c_size,
                dst_flushed,
            )
        }
    }

    #[test]
    fn output_publication_handles_no_output_space() {
        let mut output = [0xa5; 8];
        let original = output;
        let job = [1, 2, 3, 4];
        let output_size = output.len();

        let result = run_output_publication(
            &mut output,
            output_size,
            output_size,
            &job,
            job.len(),
            job.len(),
            0,
        );

        assert_eq!(result.status, FLUSH_PUBLICATION_OK);
        assert_eq!(result.toFlush, 0);
        assert_eq!(result.outputPos, output.len());
        assert_eq!(result.dstFlushed, 0);
        assert_eq!(output, original);
    }

    #[test]
    fn output_publication_handles_partial_flush_with_offsets() {
        let mut output = [0xa5; 10];
        let job = [0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17];

        let result = run_output_publication(&mut output, 9, 2, &job, job.len(), 6, 1);

        assert_eq!(result.status, FLUSH_PUBLICATION_OK);
        assert_eq!(result.toFlush, 5);
        assert_eq!(result.outputPos, 7);
        assert_eq!(result.dstFlushed, 6);
        assert_eq!(&output[..2], &[0xa5, 0xa5]);
        assert_eq!(&output[2..7], &job[1..6]);
        assert_eq!(&output[7..], &[0xa5, 0xa5, 0xa5]);
    }

    #[test]
    fn output_publication_handles_complete_flush() {
        let mut output = [0xa5; 8];
        let job = [0x20, 0x21, 0x22, 0x23, 0x24, 0x25];
        let output_size = output.len();

        let result = run_output_publication(&mut output, output_size, 1, &job, job.len(), 6, 2);

        assert_eq!(result.status, FLUSH_PUBLICATION_OK);
        assert_eq!(result.toFlush, 4);
        assert_eq!(result.outputPos, 5);
        assert_eq!(result.dstFlushed, 6);
        assert_eq!(&output[1..5], &job[2..6]);
        assert_eq!(&output[..1], &[0xa5]);
        assert_eq!(&output[5..], &[0xa5, 0xa5, 0xa5]);
    }

    #[test]
    fn output_publication_handles_repeated_partial_flushes() {
        let mut output = [0xa5; 8];
        let job = [0x30, 0x31, 0x32, 0x33, 0x34, 0x35];

        let first = run_output_publication(&mut output, 3, 0, &job, job.len(), 6, 0);
        let second = run_output_publication(
            &mut output,
            6,
            first.outputPos,
            &job,
            job.len(),
            6,
            first.dstFlushed,
        );

        assert_eq!(first.status, FLUSH_PUBLICATION_OK);
        assert_eq!(first.toFlush, 3);
        assert_eq!(first.outputPos, 3);
        assert_eq!(first.dstFlushed, 3);
        assert_eq!(second.status, FLUSH_PUBLICATION_OK);
        assert_eq!(second.toFlush, 3);
        assert_eq!(second.outputPos, 6);
        assert_eq!(second.dstFlushed, 6);
        assert_eq!(&output[..6], &job);
        assert_eq!(&output[6..], &[0xa5, 0xa5]);
    }

    #[test]
    fn output_publication_does_not_copy_trailing_sentinels() {
        let mut output = [0xa5; 12];
        let job = [0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0xde, 0xad];
        let output_size = output.len();

        let result = run_output_publication(&mut output, output_size, 4, &job, job.len(), 5, 2);

        assert_eq!(result.status, FLUSH_PUBLICATION_OK);
        assert_eq!(result.toFlush, 3);
        assert_eq!(&output[..4], &[0xa5, 0xa5, 0xa5, 0xa5]);
        assert_eq!(&output[4..7], &job[2..5]);
        assert_eq!(&output[7..], &[0xa5, 0xa5, 0xa5, 0xa5, 0xa5]);
    }

    #[test]
    fn output_publication_rejects_invalid_bounds_without_copy() {
        let mut output = [0xa5; 8];
        let original = output;
        let job = [0x50, 0x51, 0x52, 0x53];

        let result = run_output_publication(&mut output, 4, 5, &job, job.len(), 4, 0);

        assert_eq!(result.status, FLUSH_PUBLICATION_INVALID_BOUNDS);
        assert_eq!(result.toFlush, 0);
        assert_eq!(result.outputPos, 5);
        assert_eq!(result.dstFlushed, 0);
        assert_eq!(output, original);
    }

    #[test]
    fn create_job_rejects_full_table_without_callbacks() {
        let projection = ZSTDMT_createJobProjection {
            doneJobID: 0,
            nextJobID: 8,
            jobIDMask: 7,
            ..ZSTDMT_createJobProjection::default()
        };
        let result = create_compression_job_with(
            projection,
            |_job_id, _initialization| panic!("table-full job must not be prepared"),
            |_job_id| panic!("table-full job must not be written"),
            |_job_id| panic!("table-full job must not be posted"),
        );

        assert_eq!(result.action, CREATE_JOB_TABLE_FULL);
        assert_eq!(result.jobID, 0);
        assert_eq!(result.jobNumber, 8);
        assert_eq!(result.nextJobID, 8);
        assert_eq!(result.jobReady, 0);
    }

    #[test]
    fn create_job_projects_ordinary_initialization_and_prefix_advance() {
        let source = [0u8; 8];
        let prefix = [1u8; 3];
        let projection = ZSTDMT_createJobProjection {
            doneJobID: 0,
            nextJobID: 3,
            jobIDMask: 7,
            srcStart: source.as_ptr().cast(),
            srcSize: source.len(),
            inBuffFilled: source.len(),
            prefixStart: prefix.as_ptr().cast(),
            prefixSize: prefix.len(),
            targetPrefixSize: 4,
            checksumFlag: 1,
            ..ZSTDMT_createJobProjection::default()
        };
        let mut initialization = None;
        let mut posted = None;
        let result = create_compression_job_with(
            projection,
            |job_id, value| {
                assert_eq!(job_id, 3);
                initialization = Some(*value);
            },
            |_job_id| panic!("ordinary job must not use the empty-block path"),
            |job_id| {
                posted = Some(job_id);
                true
            },
        );

        let initialization = initialization.expect("job should be initialized");
        assert_eq!(initialization.srcStart, source.as_ptr().cast());
        assert_eq!(initialization.srcSize, source.len());
        assert_eq!(initialization.prefixStart, prefix.as_ptr().cast());
        assert_eq!(initialization.prefixSize, prefix.len());
        assert_eq!(
            initialization.nextPrefixStart,
            source.as_ptr().wrapping_add(4).cast()
        );
        assert_eq!(initialization.nextPrefixSize, 4);
        assert_eq!(initialization.roundBuffPosDelta, source.len());
        assert_eq!(initialization.jobNumber, 3);
        assert_eq!(initialization.firstJob, 0);
        assert_eq!(initialization.lastJob, 0);
        assert_eq!(initialization.frameChecksumNeeded, 0);
        assert_eq!(posted, Some(3));
        assert_eq!(result.action, CREATE_JOB_POST);
        assert_eq!(result.jobID, 3);
        assert_eq!(result.nextJobID, 4);
        assert_eq!(result.jobReady, 0);
    }

    #[test]
    fn create_job_projects_terminal_empty_frame() {
        let prefix = [2u8; 2];
        let projection = ZSTDMT_createJobProjection {
            doneJobID: 0,
            nextJobID: 2,
            jobIDMask: 7,
            inBuffFilled: 0,
            prefixStart: prefix.as_ptr().cast(),
            prefixSize: prefix.len(),
            endFrame: 1,
            checksumFlag: 1,
            ..ZSTDMT_createJobProjection::default()
        };
        let mut initialization = None;
        let mut empty_job = None;
        let result = create_compression_job_with(
            projection,
            |job_id, value| {
                assert_eq!(job_id, 2);
                initialization = Some(*value);
            },
            |job_id| empty_job = Some(job_id),
            |_job_id| panic!("terminal empty job must not be posted"),
        );

        let initialization = initialization.expect("empty job should be initialized");
        assert_eq!(initialization.srcSize, 0);
        assert_eq!(initialization.prefixStart, prefix.as_ptr().cast());
        assert_eq!(initialization.prefixSize, prefix.len());
        assert!(initialization.nextPrefixStart.is_null());
        assert_eq!(initialization.nextPrefixSize, 0);
        assert_eq!(initialization.roundBuffPosDelta, 0);
        assert_eq!(initialization.jobNumber, 2);
        assert_eq!(initialization.firstJob, 0);
        assert_eq!(initialization.lastJob, 1);
        assert_eq!(initialization.frameChecksumNeeded, 1);
        assert_eq!(initialization.clearChecksumFlag, 0);
        assert_eq!(empty_job, Some(2));
        assert_eq!(result.action, CREATE_JOB_EMPTY);
        assert_eq!(result.nextJobID, 3);
        assert_eq!(result.jobReady, 0);
    }

    #[test]
    fn compression_job_frame_state_classifies_all_flag_combinations() {
        for (last_job, clear_checksum_flag, expected) in [
            (0, 0, CompressionJobFrameStateAction::None),
            (0, 1, CompressionJobFrameStateAction::None),
            (1, 0, CompressionJobFrameStateAction::SetFrameEnded),
            (
                1,
                1,
                CompressionJobFrameStateAction::SetFrameEndedAndClearChecksum,
            ),
        ] {
            assert_eq!(
                classify_compression_job_frame_state(last_job, clear_checksum_flag),
                expected
            );
        }
    }

    #[test]
    fn job_preparation_orders_descriptor_input_and_terminal_state() {
        let initialization = ZSTDMT_jobInitialization {
            jobNumber: 7,
            lastJob: 2,
            clearChecksumFlag: 1,
            ..ZSTDMT_jobInitialization::default()
        };
        let events = RefCell::new(Vec::new());

        prepare_compression_job_with(
            3,
            &initialization,
            |job_id, value| {
                assert_eq!(job_id, 3);
                assert_eq!(value.jobNumber, 7);
                events.borrow_mut().push("descriptor");
            },
            |job_id, value| {
                assert_eq!(job_id, 3);
                assert_eq!(value.roundBuffPosDelta, 0);
                events.borrow_mut().push("input");
            },
            || events.borrow_mut().push("frame-ended"),
            || events.borrow_mut().push("checksum"),
        );

        assert_eq!(
            events.into_inner(),
            vec!["descriptor", "input", "frame-ended", "checksum"]
        );
    }

    #[test]
    fn job_preparation_skips_frame_state_for_nonterminal_job() {
        let initialization = ZSTDMT_jobInitialization::default();
        let events = RefCell::new(Vec::new());

        prepare_compression_job_with(
            4,
            &initialization,
            |_job_id, _value| events.borrow_mut().push("descriptor"),
            |_job_id, _value| events.borrow_mut().push("input"),
            || panic!("nonterminal job must not set frame-ended state"),
            || panic!("nonterminal job must not clear checksum state"),
        );

        assert_eq!(events.into_inner(), vec!["descriptor", "input"]);
    }

    #[test]
    fn create_job_preserves_pool_success_and_failure_state() {
        let projection = ZSTDMT_createJobProjection {
            doneJobID: 0,
            nextJobID: 4,
            jobIDMask: 7,
            jobReady: 1,
            ..ZSTDMT_createJobProjection::default()
        };
        let success = create_compression_job_with(
            projection,
            |_job_id, _initialization| panic!("prepared job must not be reinitialized"),
            |_job_id| panic!("prepared job must not use the empty-block path"),
            |job_id| {
                assert_eq!(job_id, 4);
                true
            },
        );
        assert_eq!(success.action, CREATE_JOB_POST);
        assert_eq!(success.nextJobID, 5);
        assert_eq!(success.jobReady, 0);

        let failure = create_compression_job_with(
            projection,
            |_job_id, _initialization| panic!("prepared job must not be reinitialized"),
            |_job_id| panic!("prepared job must not use the empty-block path"),
            |job_id| {
                assert_eq!(job_id, 4);
                false
            },
        );
        assert_eq!(failure.action, CREATE_JOB_POST);
        assert_eq!(failure.nextJobID, 4);
        assert_eq!(failure.jobReady, 1);
    }

    #[test]
    fn flush_state_machine_preserves_offsets_and_completes_job() {
        let job = [0x60u8, 0x61, 0x62, 0x63, 0x64, 0x65];
        let mut output = [0xa5u8; 8];
        let projection = ZSTDMT_flushJobProjection {
            consumed: 4,
            cSize: job.len(),
            srcSize: 4,
            dstStart: job.as_ptr().cast(),
            dstCapacity: job.len(),
            dstFlushed: 2,
            frameChecksumNeeded: 0,
        };
        let context = ZSTDMT_flushContextProjection {
            doneJobID: 0,
            nextJobID: 1,
            jobIDMask: 0,
            ..ZSTDMT_flushContextProjection::default()
        };
        let mut updated = None;
        let mut completed = None;

        let result = unsafe {
            flush_produced_with(
                context,
                output.as_mut_ptr().cast(),
                output.len(),
                1,
                0,
                1,
                |job_id, block_to_flush| {
                    assert_eq!(job_id, 0);
                    assert_eq!(block_to_flush, 0);
                    projection
                },
                |_job_id, _projection| panic!("unexpected checksum callback"),
                |_job_id, dst_flushed| updated = Some(dst_flushed),
                |_job_id, src_size, c_size| completed = Some((src_size, c_size)),
                || panic!("unexpected wait callback"),
                || panic!("unexpected release callback"),
            )
        };

        assert_eq!(result.result, 0);
        assert_eq!(result.outputPos, 5);
        assert_eq!(result.updateAllJobsCompleted, 1);
        assert_eq!(result.allJobsCompleted, 0);
        assert_eq!(updated, Some(6));
        assert_eq!(completed, Some((4, 6)));
        assert_eq!(&output[1..5], &job[2..]);
        assert_eq!(&output[..1], &[0xa5]);
    }

    #[test]
    fn flush_state_machine_keeps_pending_output_without_space() {
        let job = [0x70u8, 0x71, 0x72, 0x73, 0x74, 0x75];
        let mut output = [0xa5u8; 4];
        let original = output;
        let projection = ZSTDMT_flushJobProjection {
            consumed: 2,
            cSize: job.len(),
            srcSize: 5,
            dstStart: job.as_ptr().cast(),
            dstCapacity: job.len(),
            dstFlushed: 2,
            frameChecksumNeeded: 0,
        };
        let context = ZSTDMT_flushContextProjection {
            doneJobID: 3,
            nextJobID: 4,
            jobIDMask: 0,
            ..ZSTDMT_flushContextProjection::default()
        };
        let mut updated = None;
        let mut completed = false;

        let result = unsafe {
            flush_produced_with(
                context,
                output.as_mut_ptr().cast(),
                output.len(),
                output.len(),
                1,
                1,
                |_job_id, _block_to_flush| projection,
                |_job_id, _projection| panic!("unexpected checksum callback"),
                |_job_id, dst_flushed| updated = Some(dst_flushed),
                |_job_id, _src_size, _c_size| completed = true,
                || panic!("unexpected wait callback"),
                || panic!("unexpected release callback"),
            )
        };

        assert_eq!(result.result, 4);
        assert_eq!(result.outputPos, output.len());
        assert_eq!(result.updateAllJobsCompleted, 0);
        assert_eq!(updated, Some(2));
        assert!(!completed);
        assert_eq!(output, original);
    }

    #[test]
    fn flush_state_machine_normalizes_worker_error_and_checksum_projection() {
        let error_projection = ZSTDMT_flushJobProjection {
            cSize: ERROR(ZstdErrorCode::Generic),
            ..ZSTDMT_flushJobProjection::default()
        };
        let context = ZSTDMT_flushContextProjection {
            doneJobID: 0,
            nextJobID: 1,
            jobIDMask: 0,
            ..ZSTDMT_flushContextProjection::default()
        };
        let error_cleanup = Rc::new(RefCell::new(Vec::new()));
        let wait_cleanup = Rc::clone(&error_cleanup);
        let release_cleanup = Rc::clone(&error_cleanup);
        let result = unsafe {
            flush_produced_with(
                context,
                ptr::null_mut(),
                0,
                0,
                1,
                1,
                |_job_id, _block_to_flush| error_projection,
                |_job_id, _projection| panic!("unexpected checksum callback"),
                |_job_id, _dst_flushed| panic!("unexpected update callback"),
                |_job_id, _src_size, _c_size| panic!("unexpected completion callback"),
                move || wait_cleanup.borrow_mut().push("wait"),
                move || release_cleanup.borrow_mut().push("release"),
            )
        };
        assert_eq!(result.result, ERROR(ZstdErrorCode::Generic));
        assert_eq!(result.outputPos, 0);
        assert_eq!(&*error_cleanup.borrow(), &["wait", "release"]);

        let job = [0x80u8, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86];
        let mut output = [0xa5u8; 8];
        let projection = ZSTDMT_flushJobProjection {
            consumed: 4,
            cSize: 3,
            srcSize: 4,
            dstStart: job.as_ptr().cast(),
            dstCapacity: job.len(),
            dstFlushed: 0,
            frameChecksumNeeded: 1,
        };
        let mut checksum_calls = 0;
        let mut completed_size = None;
        let result = unsafe {
            flush_produced_with(
                ZSTDMT_flushContextProjection {
                    doneJobID: 0,
                    nextJobID: 1,
                    jobIDMask: 0,
                    ..ZSTDMT_flushContextProjection::default()
                },
                output.as_mut_ptr().cast(),
                output.len(),
                0,
                0,
                1,
                |_job_id, _block_to_flush| projection,
                |_job_id, projected| {
                    checksum_calls += 1;
                    projected.cSize = 7;
                    projected.frameChecksumNeeded = 0;
                },
                |_job_id, _dst_flushed| {},
                |_job_id, _src_size, c_size| completed_size = Some(c_size),
                || panic!("unexpected wait callback"),
                || panic!("unexpected release callback"),
            )
        };
        assert_eq!(checksum_calls, 1);
        assert_eq!(completed_size, Some(7));
        assert_eq!(result.result, 0);
        assert_eq!(&output[..7], &job);
    }

    #[test]
    fn flush_state_machine_propagates_checksum_projection_error() {
        let job = [0x80u8, 0x81, 0x82, 0x83];
        let mut output = [0xa5u8; 8];
        let projection = ZSTDMT_flushJobProjection {
            consumed: job.len(),
            cSize: 3,
            srcSize: job.len(),
            dstStart: job.as_ptr().cast(),
            dstCapacity: job.len(),
            frameChecksumNeeded: 1,
            ..ZSTDMT_flushJobProjection::default()
        };
        let error_cleanup = Rc::new(RefCell::new(Vec::new()));
        let wait_cleanup = Rc::clone(&error_cleanup);
        let release_cleanup = Rc::clone(&error_cleanup);
        let mut completed = false;

        let result = unsafe {
            flush_produced_with(
                ZSTDMT_flushContextProjection {
                    doneJobID: 0,
                    nextJobID: 1,
                    jobIDMask: 0,
                    ..ZSTDMT_flushContextProjection::default()
                },
                output.as_mut_ptr().cast(),
                output.len(),
                0,
                0,
                1,
                |_job_id, _block_to_flush| projection,
                |_job_id, projected| {
                    projected.cSize = ERROR(ZstdErrorCode::DstSizeTooSmall);
                },
                |_job_id, _dst_flushed| panic!("unexpected update callback"),
                |_job_id, _src_size, _c_size| completed = true,
                move || wait_cleanup.borrow_mut().push("wait"),
                move || release_cleanup.borrow_mut().push("release"),
            )
        };

        assert_eq!(result.result, ERROR(ZstdErrorCode::DstSizeTooSmall));
        assert_eq!(result.outputPos, 0);
        assert_eq!(&*error_cleanup.borrow(), &["wait", "release"]);
        assert!(!completed);
        assert_eq!(output, [0xa5; 8]);
    }

    #[test]
    fn frame_checksum_is_serialized_little_endian() {
        let mut dst = [0xa5u8; 8];
        let projection = ZSTDMT_frameChecksumProjection {
            dst: dst.as_mut_ptr().cast(),
            dstCapacity: dst.len(),
            cSize: 4,
            checksum: 0x7856_3412,
        };

        assert_eq!(write_frame_checksum(&projection), 8);
        assert_eq!(&dst[..4], &[0xa5; 4]);
        assert_eq!(&dst[4..], &[0x12, 0x34, 0x56, 0x78]);
    }

    #[test]
    fn frame_checksum_accepts_exact_destination_capacity() {
        let mut dst = [0xa5u8; 6];
        let projection = ZSTDMT_frameChecksumProjection {
            dst: dst.as_mut_ptr().cast(),
            dstCapacity: dst.len(),
            cSize: 2,
            checksum: 0x1122_3344,
        };

        assert_eq!(write_frame_checksum(&projection), dst.len());
        assert_eq!(&dst, &[0xa5, 0xa5, 0x44, 0x33, 0x22, 0x11]);
    }

    #[test]
    fn frame_checksum_rejects_too_small_or_null_destination() {
        let mut dst = [0xa5u8; 8];
        let too_small = ZSTDMT_frameChecksumProjection {
            dst: dst.as_mut_ptr().cast(),
            dstCapacity: 7,
            cSize: 4,
            checksum: 0x0102_0304,
        };
        let null_destination = ZSTDMT_frameChecksumProjection {
            dst: ptr::null_mut(),
            dstCapacity: 4,
            cSize: 0,
            checksum: 0x0102_0304,
        };

        assert_eq!(
            write_frame_checksum(&too_small),
            ERROR(ZstdErrorCode::DstSizeTooSmall)
        );
        assert_eq!(
            write_frame_checksum(&null_destination),
            ERROR(ZstdErrorCode::DstSizeTooSmall)
        );
        assert_eq!(dst, [0xa5; 8]);
    }

    #[test]
    fn flush_state_machine_reports_end_of_frame_state() {
        let not_ended = unsafe {
            flush_produced_with(
                ZSTDMT_flushContextProjection {
                    frameEnded: 0,
                    ..ZSTDMT_flushContextProjection::default()
                },
                ptr::null_mut(),
                0,
                0,
                0,
                ZSTD_E_END,
                |_job_id, _block_to_flush| panic!("no job expected"),
                |_job_id, _projection| panic!("no checksum expected"),
                |_job_id, _dst_flushed| panic!("no update expected"),
                |_job_id, _src_size, _c_size| panic!("no completion expected"),
                || panic!("no wait expected"),
                || panic!("no release expected"),
            )
        };
        assert_eq!(not_ended.result, 1);
        assert_eq!(not_ended.updateAllJobsCompleted, 1);
        assert_eq!(not_ended.allJobsCompleted, 0);

        let ended = unsafe {
            flush_produced_with(
                ZSTDMT_flushContextProjection {
                    frameEnded: 1,
                    ..ZSTDMT_flushContextProjection::default()
                },
                ptr::null_mut(),
                0,
                0,
                0,
                ZSTD_E_END,
                |_job_id, _block_to_flush| panic!("no job expected"),
                |_job_id, _projection| panic!("no checksum expected"),
                |_job_id, _dst_flushed| panic!("no update expected"),
                |_job_id, _src_size, _c_size| panic!("no completion expected"),
                || panic!("no wait expected"),
                || panic!("no release expected"),
            )
        };
        assert_eq!(ended.result, 0);
        assert_eq!(ended.updateAllJobsCompleted, 1);
        assert_eq!(ended.allJobsCompleted, 1);
    }

    #[test]
    fn outer_scheduler_copies_input_and_reports_filled_after_progress() {
        let input_bytes = [1u8, 2, 3, 4, 5];
        let mut round_buffer = [0xa5u8; 4];
        let mut output_buffer = [0xb6u8; 8];
        let context = ZSTDMT_compressStreamContextProjection {
            frameEnded: 0,
            jobReady: 0,
            inBuffStart: ptr::null_mut(),
            inBuffCapacity: 0,
            inBuffFilled: 0,
            targetSectionSize: round_buffer.len(),
            rsyncable: 0,
            ..ZSTDMT_compressStreamContextProjection::default()
        };
        let input = ZSTDMT_streamInputProjection {
            src: input_bytes.as_ptr().cast(),
            size: input_bytes.len(),
            pos: 0,
        };
        let output = ZSTDMT_streamOutputProjection {
            dst: output_buffer.as_mut_ptr().cast(),
            size: output_buffer.len(),
            pos: 1,
        };
        let mut range_calls = 0;
        let mut create_calls = Vec::new();
        let mut flush_calls = Vec::new();
        let round_buffer_ptr = round_buffer.as_mut_ptr();
        let round_capacity = round_buffer.len();

        let result = unsafe {
            compress_stream_generic_with(
                context,
                input,
                output,
                ZSTD_E_CONTINUE,
                |projection| {
                    range_calls += 1;
                    projection.bufferStart = round_buffer_ptr.cast();
                    projection.bufferCapacity = round_capacity;
                    projection.bufferFilled = 0;
                    1
                },
                |src_size, end| {
                    create_calls.push((src_size, end));
                    ZSTDMT_streamCreateJobResult {
                        result: 0,
                        inBuffActive: 1,
                    }
                },
                |_dst, _size, output_pos, block_to_flush, end| {
                    flush_calls.push((block_to_flush, end));
                    ZSTDMT_streamFlushResult {
                        result: 0,
                        outputPos: output_pos + 2,
                    }
                },
                |_context, input, projection| ZSTDMT_syncPointProjection {
                    toLoad: (input.size - input.pos).min(round_capacity - projection.bufferFilled),
                    flush: 0,
                },
            )
        };

        assert_eq!(range_calls, 1);
        assert_eq!(round_buffer, input_bytes[..round_buffer.len()]);
        assert_eq!(create_calls, vec![(round_buffer.len(), ZSTD_E_CONTINUE)]);
        assert_eq!(flush_calls, vec![(0, ZSTD_E_CONTINUE)]);
        assert_eq!(result.result, 1);
        assert_eq!(result.inputPos, round_buffer.len());
        assert_eq!(result.outputPos, 3);
        assert_eq!(result.inBuffFilled, round_buffer.len());
        assert_eq!(output_buffer[0], 0xb6);
    }

    #[test]
    fn outer_scheduler_suppresses_filled_after_job_clears_range() {
        let mut round_buffer = [0xa5u8; 4];
        let context = ZSTDMT_compressStreamContextProjection {
            inBuffStart: round_buffer.as_mut_ptr().cast(),
            inBuffCapacity: round_buffer.len(),
            inBuffFilled: 2,
            targetSectionSize: round_buffer.len(),
            ..ZSTDMT_compressStreamContextProjection::default()
        };
        let mut create_call = None;

        let result = unsafe {
            compress_stream_generic_with(
                context,
                ZSTDMT_streamInputProjection::default(),
                ZSTDMT_streamOutputProjection::default(),
                ZSTD_E_FLUSH,
                |_projection| panic!("the active range must not be reacquired"),
                |src_size, end| {
                    create_call = Some((src_size, end));
                    ZSTDMT_streamCreateJobResult {
                        result: 0,
                        inBuffActive: 0,
                    }
                },
                |_dst, _size, output_pos, _block_to_flush, _end| ZSTDMT_streamFlushResult {
                    result: 0,
                    outputPos: output_pos,
                },
                |_context, _input, _projection| panic!("no synchronization scan expected"),
            )
        };

        assert_eq!(create_call, Some((2, ZSTD_E_FLUSH)));
        assert_eq!(result.result, 0);
        assert_eq!(result.inBuffFilled, 0);
    }

    #[test]
    fn outer_scheduler_forces_flush_before_end_when_input_remains() {
        let input_bytes = [7u8, 8, 9, 10, 11];
        let mut round_buffer = [0xa5u8; 3];
        let context = ZSTDMT_compressStreamContextProjection {
            inBuffStart: round_buffer.as_mut_ptr().cast(),
            inBuffCapacity: round_buffer.len(),
            targetSectionSize: round_buffer.len(),
            rsyncable: 0,
            ..ZSTDMT_compressStreamContextProjection::default()
        };
        let input = ZSTDMT_streamInputProjection {
            src: input_bytes.as_ptr().cast(),
            size: input_bytes.len(),
            pos: 0,
        };
        let output = ZSTDMT_streamOutputProjection::default();
        let mut create_end = None;
        let mut flush_end = None;
        let mut flush_block = None;

        let result = unsafe {
            compress_stream_generic_with(
                context,
                input,
                output,
                ZSTD_E_END,
                |_projection| panic!("input range should already be available"),
                |src_size, end| {
                    create_end = Some((src_size, end));
                    ZSTDMT_streamCreateJobResult {
                        result: 0,
                        inBuffActive: 0,
                    }
                },
                |_dst, _size, output_pos, block_to_flush, end| {
                    flush_block = Some(block_to_flush);
                    flush_end = Some(end);
                    ZSTDMT_streamFlushResult {
                        result: 0,
                        outputPos: output_pos,
                    }
                },
                |_context, input, projection| ZSTDMT_syncPointProjection {
                    toLoad: (input.size - input.pos)
                        .min(round_buffer.len() - projection.bufferFilled),
                    flush: 0,
                },
            )
        };

        assert_eq!(create_end, Some((round_buffer.len(), ZSTD_E_FLUSH)));
        assert_eq!(flush_end, Some(ZSTD_E_FLUSH));
        assert_eq!(flush_block, Some(0));
        assert_eq!(result.result, 1);
        assert_eq!(result.inputPos, round_buffer.len());
    }

    #[test]
    fn outer_scheduler_resubmits_ready_job_and_blocks_for_flush_progress() {
        let context = ZSTDMT_compressStreamContextProjection {
            jobReady: 1,
            targetSectionSize: 4,
            ..ZSTDMT_compressStreamContextProjection::default()
        };
        let input_bytes = [0x31u8, 0x32];
        let input = ZSTDMT_streamInputProjection {
            src: input_bytes.as_ptr().cast(),
            size: input_bytes.len(),
            pos: 0,
        };
        let output = ZSTDMT_streamOutputProjection::default();
        let mut create_call = None;
        let mut flush_call = None;

        let result = unsafe {
            compress_stream_generic_with(
                context,
                input,
                output,
                ZSTD_E_CONTINUE,
                |_projection| panic!("a ready job must skip input acquisition"),
                |src_size, end| {
                    create_call = Some((src_size, end));
                    ZSTDMT_streamCreateJobResult {
                        result: 0,
                        inBuffActive: 0,
                    }
                },
                |_dst, _size, output_pos, block_to_flush, end| {
                    flush_call = Some((block_to_flush, end));
                    ZSTDMT_streamFlushResult {
                        result: 1,
                        outputPos: output_pos,
                    }
                },
                |_context, _input, _projection| panic!("ready job must skip sync scan"),
            )
        };

        assert_eq!(create_call, Some((0, ZSTD_E_CONTINUE)));
        assert_eq!(flush_call, Some((1, ZSTD_E_CONTINUE)));
        assert_eq!(result.result, 1);
        assert_eq!(result.inputPos, 0);
    }

    #[test]
    fn outer_scheduler_stops_before_flush_on_job_error() {
        let mut round_buffer = [0xa5u8; 2];
        let input_bytes = [0x41u8, 0x42];
        let context = ZSTDMT_compressStreamContextProjection {
            inBuffStart: round_buffer.as_mut_ptr().cast(),
            inBuffCapacity: round_buffer.len(),
            targetSectionSize: round_buffer.len(),
            ..ZSTDMT_compressStreamContextProjection::default()
        };
        let input = ZSTDMT_streamInputProjection {
            src: input_bytes.as_ptr().cast(),
            size: input_bytes.len(),
            pos: 0,
        };
        let output = ZSTDMT_streamOutputProjection::default();
        let mut flush_called = false;

        let result = unsafe {
            compress_stream_generic_with(
                context,
                input,
                output,
                ZSTD_E_CONTINUE,
                |_projection| panic!("input range should already be available"),
                |_src_size, _end| ZSTDMT_streamCreateJobResult {
                    result: ERROR(ZstdErrorCode::DstSizeTooSmall),
                    inBuffActive: 1,
                },
                |_dst, _size, output_pos, _block_to_flush, _end| {
                    flush_called = true;
                    ZSTDMT_streamFlushResult {
                        result: 0,
                        outputPos: output_pos,
                    }
                },
                |_context, input, projection| ZSTDMT_syncPointProjection {
                    toLoad: (input.size - input.pos)
                        .min(round_buffer.len() - projection.bufferFilled),
                    flush: 0,
                },
            )
        };

        assert_eq!(result.result, ERROR(ZstdErrorCode::DstSizeTooSmall));
        assert_eq!(result.inputPos, round_buffer.len());
        assert_eq!(result.outputPos, 0);
        assert_eq!(result.inBuffFilled, round_buffer.len());
        assert!(!flush_called);
    }

    #[test]
    fn empty_last_block_serializes_after_buffer_callback() {
        let mut storage = vec![0xa5u8; 3];
        let expected_start = storage.as_mut_ptr().cast::<c_void>();
        let mut callback_calls = 0;
        let projection = ZSTDMT_emptyBlockJobProjection {
            lastJob: 1,
            srcSize: 0,
            firstJob: 0,
            dstStart: ptr::null_mut(),
            dstCapacity: 0,
            consumed: 0,
        };

        let result = unsafe {
            write_last_empty_block_with(projection, || {
                callback_calls += 1;
                ZstdMtBuffer {
                    start: expected_start,
                    capacity: storage.len(),
                }
            })
        };

        assert_eq!(callback_calls, 1);
        assert_eq!(result.buffer.start, expected_start);
        assert_eq!(result.buffer.capacity, storage.len());
        assert_eq!(result.cSize, 3);
        assert_eq!(result.clearSource, 1);
        assert_eq!(storage, [1, 0, 0]);
    }

    #[test]
    fn empty_last_block_keeps_source_on_buffer_allocation_failure() {
        let projection = ZSTDMT_emptyBlockJobProjection {
            lastJob: 1,
            srcSize: 0,
            firstJob: 0,
            dstStart: ptr::null_mut(),
            dstCapacity: 0,
            consumed: 0,
        };
        let result = unsafe { write_last_empty_block_with(projection, ZstdMtBuffer::default) };

        assert!(result.buffer.start.is_null());
        assert_eq!(result.buffer.capacity, 0);
        assert_eq!(result.cSize, ERROR(ZstdErrorCode::MemoryAllocation));
        assert_eq!(result.clearSource, 0);
    }

    #[test]
    fn compression_job_begin_uses_cdict_and_publishes_header_after_init() {
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let projection = ZSTDMT_compressionJobBeginProjection {
            firstJob: 1,
            hasCDict: 1,
        };

        let cdict_events = Rc::clone(&events);
        let header_events = Rc::clone(&events);
        let result = begin_compression_job_with(
            projection,
            move || {
                cdict_events.borrow_mut().push("cdict");
                0
            },
            |_| panic!("cdict jobs must not update force-window parameters"),
            || panic!("cdict jobs must not update deterministic-prefix parameters"),
            || panic!("cdict jobs must not initialize from a raw prefix"),
            move || header_events.borrow_mut().push("header"),
        );

        assert_eq!(result, 0);
        assert_eq!(&*events.borrow(), &["cdict", "header"]);
    }

    #[test]
    fn compression_job_begin_orders_nonfirst_parameter_policy_before_prefix_init() {
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let projection = ZSTDMT_compressionJobBeginProjection {
            firstJob: 0,
            hasCDict: 0,
        };

        let force_events = Rc::clone(&events);
        let deterministic_events = Rc::clone(&events);
        let prefix_events = Rc::clone(&events);
        let header_events = Rc::clone(&events);
        let result = begin_compression_job_with(
            projection,
            || panic!("raw-prefix jobs must not use a cdict"),
            move |value| {
                assert_eq!(value, 1);
                force_events.borrow_mut().push("force-window");
                0
            },
            move || {
                deterministic_events
                    .borrow_mut()
                    .push("deterministic-prefix");
                0
            },
            move || {
                prefix_events.borrow_mut().push("prefix");
                0
            },
            move || header_events.borrow_mut().push("header"),
        );

        assert_eq!(result, 0);
        assert_eq!(
            &*events.borrow(),
            &["force-window", "deterministic-prefix", "prefix", "header"]
        );
    }

    #[test]
    fn compression_job_begin_stops_after_force_window_error() {
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let expected_error = ERROR(ZstdErrorCode::MemoryAllocation);
        let projection = ZSTDMT_compressionJobBeginProjection {
            firstJob: 0,
            hasCDict: 0,
        };
        let force_events = Rc::clone(&events);

        let result = begin_compression_job_with(
            projection,
            || panic!("parameter failure must stop before cdict initialization"),
            move |_| {
                force_events.borrow_mut().push("force-window");
                expected_error
            },
            || panic!("parameter failure must stop before deterministic-prefix update"),
            || panic!("parameter failure must stop before prefix initialization"),
            || panic!("parameter failure must stop before header publication"),
        );

        assert_eq!(result, expected_error);
        assert_eq!(&*events.borrow(), &["force-window"]);
    }

    #[test]
    fn compression_job_begin_does_not_publish_header_after_init_error() {
        let events = Rc::new(RefCell::new(Vec::<&'static str>::new()));
        let expected_error = ERROR(ZstdErrorCode::Generic);
        let projection = ZSTDMT_compressionJobBeginProjection {
            firstJob: 1,
            hasCDict: 1,
        };
        let cdict_events = Rc::clone(&events);

        let result = begin_compression_job_with(
            projection,
            move || {
                cdict_events.borrow_mut().push("cdict");
                expected_error
            },
            |_| panic!("cdict initialization failure must not update parameters"),
            || panic!("cdict initialization failure must not update prefix policy"),
            || panic!("cdict initialization failure must not initialize a raw prefix"),
            || panic!("cdict initialization failure must not publish a header"),
        );

        assert_eq!(result, expected_error);
        assert_eq!(&*events.borrow(), &["cdict"]);
    }

    #[test]
    fn compression_job_begin_rejects_cdict_on_nonfirst_job() {
        let projection = ZSTDMT_compressionJobBeginProjection {
            firstJob: 0,
            hasCDict: 1,
        };

        let result = begin_compression_job_with(
            projection,
            || panic!("invalid cdict placement must not initialize"),
            |_| panic!("invalid cdict placement must not update parameters"),
            || panic!("invalid cdict placement must not update prefix policy"),
            || panic!("invalid cdict placement must not initialize a raw prefix"),
            || panic!("invalid cdict placement must not publish a header"),
        );

        assert_eq!(result, ERROR(ZstdErrorCode::StageWrong));
    }

    #[test]
    fn chunk_loop_handles_empty_jobs_without_compression() {
        let mut compressor = MockChunkCompressor {
            continue_inputs: Vec::new(),
            end_inputs: Vec::new(),
            continue_results: Vec::new(),
            end_results: vec![9],
        };
        let mut progress = MockChunkProgress::default();

        let result = run_mock_chunk_loop(&mut compressor, &mut progress, 0, 8, 0);
        assert_eq!(result, ZSTDMT_chunkProcessResult::default());
        assert!(compressor.continue_inputs.is_empty());
        assert!(compressor.end_inputs.is_empty());
        assert!(progress.calls.is_empty());

        let result = run_mock_chunk_loop(&mut compressor, &mut progress, 0, 8, 1);
        assert_eq!(result.error, 0);
        assert_eq!(result.lastBlockSize, 9);
        assert!(compressor.continue_inputs.is_empty());
        assert_eq!(compressor.end_inputs, [0]);
        assert!(progress.calls.is_empty());
    }

    #[test]
    fn chunk_loop_handles_one_exact_chunk_and_last_job_dispatch() {
        let mut compressor = MockChunkCompressor {
            continue_inputs: Vec::new(),
            end_inputs: Vec::new(),
            continue_results: vec![4],
            end_results: vec![6],
        };
        let mut progress = MockChunkProgress::default();

        let result = run_mock_chunk_loop(&mut compressor, &mut progress, 8, 8, 0);
        assert_eq!(result.lastBlockSize, 4);
        assert_eq!(compressor.continue_inputs, [8]);
        assert!(compressor.end_inputs.is_empty());
        assert!(progress.calls.is_empty());

        let result = run_mock_chunk_loop(&mut compressor, &mut progress, 8, 8, 1);
        assert_eq!(result.lastBlockSize, 6);
        assert_eq!(compressor.end_inputs, [8]);
        assert!(progress.calls.is_empty());
    }

    #[test]
    fn chunk_loop_handles_multiple_exact_chunks_and_orders_progress() {
        let mut compressor = MockChunkCompressor {
            continue_inputs: Vec::new(),
            end_inputs: Vec::new(),
            continue_results: vec![2, 3],
            end_results: Vec::new(),
        };
        let mut progress = MockChunkProgress::default();

        let result = run_mock_chunk_loop(&mut compressor, &mut progress, 16, 8, 0);
        assert_eq!(result.lastBlockSize, 3);
        assert_eq!(compressor.continue_inputs, [8, 8]);
        assert!(compressor.end_inputs.is_empty());
        assert_eq!(progress.calls, [(2, 8)]);
    }

    #[test]
    fn chunk_loop_handles_partial_tail_and_final_dispatch() {
        let mut compressor = MockChunkCompressor {
            continue_inputs: Vec::new(),
            end_inputs: Vec::new(),
            continue_results: vec![2],
            end_results: vec![5],
        };
        let mut progress = MockChunkProgress::default();

        let result = run_mock_chunk_loop(&mut compressor, &mut progress, 11, 8, 1);
        assert_eq!(result.lastBlockSize, 5);
        assert_eq!(compressor.continue_inputs, [8]);
        assert_eq!(compressor.end_inputs, [3]);
        assert_eq!(progress.calls, [(2, 8)]);
    }

    #[test]
    fn chunk_loop_propagates_intermediate_errors_after_prior_progress() {
        let error = ERROR(ZstdErrorCode::DstSizeTooSmall);
        let mut compressor = MockChunkCompressor {
            continue_inputs: Vec::new(),
            end_inputs: Vec::new(),
            continue_results: vec![2, error],
            end_results: vec![7],
        };
        let mut progress = MockChunkProgress::default();

        let result = run_mock_chunk_loop(&mut compressor, &mut progress, 24, 8, 1);
        assert_eq!(result.error, error);
        assert_eq!(result.lastBlockSize, 0);
        assert_eq!(compressor.continue_inputs, [8, 8]);
        assert!(compressor.end_inputs.is_empty());
        assert_eq!(progress.calls, [(2, 8)]);
    }

    #[test]
    fn chunk_loop_propagates_final_errors_without_progress_callback() {
        let error = ERROR(ZstdErrorCode::Generic);
        let mut compressor = MockChunkCompressor {
            continue_inputs: Vec::new(),
            end_inputs: Vec::new(),
            continue_results: Vec::new(),
            end_results: vec![error],
        };
        let mut progress = MockChunkProgress::default();

        let result = run_mock_chunk_loop(&mut compressor, &mut progress, 3, 8, 1);
        assert_eq!(result.error, error);
        assert_eq!(result.lastBlockSize, 0);
        assert!(compressor.continue_inputs.is_empty());
        assert_eq!(compressor.end_inputs, [3]);
        assert!(progress.calls.is_empty());
    }

    unsafe extern "C" fn probe_job_table_init(
        _job_table: *mut c_void,
        _nb_jobs: c_uint,
        _job_size: usize,
    ) -> c_int {
        JOB_TABLE_INIT_CALLS.fetch_add(1, Ordering::Relaxed);
        if JOB_TABLE_FAIL_INIT.load(Ordering::Relaxed) {
            -1
        } else {
            0
        }
    }

    unsafe extern "C" fn probe_job_table_destroy(
        _job_table: *mut c_void,
        _nb_jobs: c_uint,
        _job_size: usize,
    ) {
        JOB_TABLE_DESTROY_CALLS.fetch_add(1, Ordering::Relaxed);
    }

    #[derive(Default)]
    struct FreeJobsTableTestContext {
        events: Vec<&'static str>,
    }

    unsafe extern "C" fn free_jobs_table_test_alloc(
        _opaque: *mut c_void,
        _size: usize,
    ) -> *mut c_void {
        ptr::null_mut()
    }

    unsafe extern "C" fn free_jobs_table_test_destroy(
        job_table: *mut c_void,
        _nb_jobs: c_uint,
        _job_size: usize,
    ) {
        unsafe {
            (*job_table.cast::<FreeJobsTableTestContext>())
                .events
                .push("destroy-sync");
        }
    }

    unsafe extern "C" fn free_jobs_table_test_free(opaque: *mut c_void, allocation: *mut c_void) {
        assert_eq!(allocation, opaque);
        unsafe {
            (*opaque.cast::<FreeJobsTableTestContext>())
                .events
                .push("free-storage");
        }
    }

    #[test]
    fn free_jobs_table_orders_sync_destruction_before_storage_release() {
        let mut context = FreeJobsTableTestContext::default();
        let context_ptr = (&mut context as *mut FreeJobsTableTestContext).cast::<c_void>();
        let projection = ZSTDMT_RustFreeJobsTableProjection {
            jobTable: context_ptr,
            nbJobs: 8,
            jobSize: 128,
            customMem: ZstdCustomMem {
                customAlloc: Some(free_jobs_table_test_alloc),
                customFree: Some(free_jobs_table_test_free),
                opaque: context_ptr,
            },
            destroySync: Some(free_jobs_table_test_destroy),
        };

        unsafe { ZSTDMT_rust_freeJobsTable(&projection) };

        assert_eq!(context.events, ["destroy-sync", "free-storage"]);
    }

    fn rsync_prime_power() -> u64 {
        (0..RSYNC_LENGTH - 1).fold(1, |power, _| power.wrapping_mul(PRIME8_BYTES))
    }

    fn patterned_bytes(length: usize, seed: u8) -> Vec<u8> {
        let mut value = seed;
        let mut bytes = vec![0; length];
        for byte in &mut bytes {
            value = value.wrapping_mul(17).wrapping_add(29);
            *byte = value;
        }
        bytes
    }

    fn rolling_hash_bytes(bytes: &[u8]) -> u64 {
        bytes.iter().fold(0, |hash, &byte| {
            hash.wrapping_mul(PRIME8_BYTES)
                .wrapping_add(u64::from(byte).wrapping_add(ROLL_HASH_CHAR_OFFSET))
        })
    }

    #[test]
    fn rolling_hash_prime_power_matches_c_ipow_for_common_lengths() {
        assert_eq!(ZSTDMT_rust_rollingHashPrimePower(1), 0x0000_0000_0000_0001);
        assert_eq!(ZSTDMT_rust_rollingHashPrimePower(2), 0xCF1B_BCDC_B7A5_6463);
        assert_eq!(
            ZSTDMT_rust_rollingHashPrimePower(RSYNC_LENGTH as c_uint),
            0xF550_7FE3_5F91_F8CB
        );
    }

    #[test]
    fn rolling_hash_prime_power_preserves_zero_length_underflow() {
        assert_eq!(0u32.wrapping_sub(1), c_uint::MAX);
        assert_eq!(ZSTDMT_rust_rollingHashPrimePower(0), 0x12A9_3A33_31E0_3D4B);
    }

    #[test]
    fn frame_progression_zero_jobs_keeps_mt_base_values() {
        assert_eq!(
            ZSTDMT_rust_frameProgression(11, 7, 13, 4),
            ZSTD_frameProgression {
                ingested: 18,
                consumed: 11,
                produced: 13,
                flushed: 13,
                currentJobID: 4,
                nbActiveWorkers: 0,
            }
        );
    }

    #[test]
    fn frame_progression_one_ready_job_counts_as_active() {
        let base = ZSTDMT_rust_frameProgression(100, 20, 300, 9);
        assert_eq!(
            ZSTDMT_rust_frameProgressionAddJob(base, 80, 17, 42, 10),
            ZSTD_frameProgression {
                ingested: 200,
                consumed: 117,
                produced: 342,
                flushed: 310,
                currentJobID: 9,
                nbActiveWorkers: 1,
            }
        );
    }

    #[test]
    fn frame_progression_completed_job_is_not_active() {
        let base = ZSTDMT_rust_frameProgression(100, 20, 300, 9);
        let progression = ZSTDMT_rust_frameProgressionAddJob(base, 80, 80, 42, 42);

        assert_eq!(progression.nbActiveWorkers, 0);
        assert_eq!(progression.ingested, 200);
        assert_eq!(progression.consumed, 180);
    }

    #[test]
    fn frame_progression_error_job_uses_normalized_zero_output() {
        let base = ZSTDMT_rust_frameProgression(100, 20, 300, 9);
        let progression = ZSTDMT_rust_frameProgressionAddJob(base, 80, 0, 0, 0);

        assert_eq!(progression.produced, 300);
        assert_eq!(progression.flushed, 300);
        assert_eq!(progression.nbActiveWorkers, 1);
    }

    #[test]
    fn frame_progression_aggregates_active_and_completed_jobs() {
        let base = ZSTDMT_rust_frameProgression(100, 20, 300, 9);
        let progression = ZSTDMT_rust_frameProgressionAddJob(base, 80, 17, 42, 10);
        let progression = ZSTDMT_rust_frameProgressionAddJob(progression, 30, 30, 8, 8);
        let progression = ZSTDMT_rust_frameProgressionAddJob(progression, 40, 5, 7, 6);

        assert_eq!(
            progression,
            ZSTD_frameProgression {
                ingested: 270,
                consumed: 152,
                produced: 357,
                flushed: 324,
                currentJobID: 9,
                nbActiveWorkers: 2,
            }
        );
    }

    #[test]
    fn frame_progression_with_jobs_masks_slots_and_normalizes_errors() {
        let projections = [
            ZSTDMT_jobProjection {
                consumed: 50,
                cSize: 8,
                srcSize: 50,
                dstFlushed: 8,
                ..ZSTDMT_jobProjection::default()
            },
            ZSTDMT_jobProjection {
                consumed: 17,
                cSize: 42,
                srcSize: 80,
                dstFlushed: 10,
                ..ZSTDMT_jobProjection::default()
            },
            ZSTDMT_jobProjection {
                consumed: 30,
                cSize: ERROR(ZstdErrorCode::SequenceProducerFailed),
                srcSize: 30,
                dstFlushed: 29,
                ..ZSTDMT_jobProjection::default()
            },
            ZSTDMT_jobProjection {
                consumed: 5,
                cSize: 7,
                srcSize: 40,
                dstFlushed: 6,
                ..ZSTDMT_jobProjection::default()
            },
        ];
        let mut calls = Vec::new();

        let progression = frame_progression_with_jobs(100, 20, 300, 9, 5, 8, 1, 3, |job_id| {
            calls.push(job_id);
            projections[job_id as usize]
        });

        assert_eq!(calls, [1, 2, 3, 0]);
        assert_eq!(
            progression,
            ZSTD_frameProgression {
                ingested: 320,
                consumed: 202,
                produced: 357,
                flushed: 324,
                currentJobID: 9,
                nbActiveWorkers: 2,
            }
        );
    }

    #[test]
    fn input_data_in_use_skips_first_round_and_scans_oldest_ring_slot() {
        let source_a = [0u8; 8];
        let backing = [1u8; 16];
        let source_b = &backing[8..];
        let prefix_b = &backing[..3];
        let projections = [
            ZSTDMT_jobProjection {
                consumed: source_a.len(),
                srcStart: source_a.as_ptr().cast(),
                srcSize: source_a.len(),
                ..ZSTDMT_jobProjection::default()
            },
            ZSTDMT_jobProjection {
                consumed: 2,
                srcStart: source_b.as_ptr().cast(),
                srcSize: source_b.len(),
                prefixStart: prefix_b.as_ptr().cast(),
                prefixSize: prefix_b.len(),
                ..ZSTDMT_jobProjection::default()
            },
        ];

        let mut calls = 0;
        let result = get_input_data_in_use_with(4, 6, 1, 16, 8, |slot| {
            calls += 1;
            projections[slot as usize]
        });

        assert_eq!(calls, 2);
        assert_eq!(result.start, prefix_b.as_ptr().cast());
        assert_eq!(result.size, prefix_b.len());

        let mut first_round_calls = 0;
        let result = get_input_data_in_use_with(0, 3, 1, 32, 8, |slot| {
            first_round_calls += 1;
            projections[slot as usize]
        });
        assert_eq!(first_round_calls, 0);
        assert_eq!(result, ZSTDMT_inputRange::default());
    }

    #[test]
    fn input_data_in_use_falls_back_to_source_without_prefix() {
        let source = [3u8; 11];
        let projection = ZSTDMT_jobProjection {
            consumed: 4,
            srcStart: source.as_ptr().cast(),
            srcSize: source.len(),
            ..ZSTDMT_jobProjection::default()
        };

        let result = get_input_data_in_use_with(7, 8, 7, 8, 8, |_| projection);
        assert_eq!(result.start, source.as_ptr().cast());
        assert_eq!(result.size, source.len());
    }

    #[test]
    fn try_get_input_range_selects_next_section_without_wrapping() {
        let round_buffer = [0u8; 32];
        let prefix = [1u8; 4];
        let projection = ZSTDMT_tryGetInputRangeProjection {
            roundBufferStart: round_buffer.as_ptr().cast(),
            roundBufferCapacity: round_buffer.len(),
            roundBufferPos: 8,
            prefixStart: prefix.as_ptr().cast(),
            prefixSize: prefix.len(),
            targetSectionSize: 8,
            ..ZSTDMT_tryGetInputRangeProjection::default()
        };

        let result = try_get_input_range_with(projection);

        assert_eq!(result.ready, 1);
        assert_eq!(result.movePrefix, 0);
        assert_eq!(result.roundBufferPos, 8);
        assert_eq!(
            result.bufferStart,
            round_buffer.as_ptr().wrapping_add(8).cast_mut().cast()
        );
        assert_eq!(result.bufferCapacity, 8);
    }

    #[test]
    fn try_get_input_range_wraps_after_prefix_and_rejects_live_ranges() {
        let round_buffer = [0u8; 32];
        let prefix = [1u8; 4];
        let wrapped = ZSTDMT_tryGetInputRangeProjection {
            roundBufferStart: round_buffer.as_ptr().cast(),
            roundBufferCapacity: round_buffer.len(),
            roundBufferPos: 28,
            prefixStart: prefix.as_ptr().cast(),
            prefixSize: prefix.len(),
            targetSectionSize: 8,
            ..ZSTDMT_tryGetInputRangeProjection::default()
        };
        let result = try_get_input_range_with(wrapped);
        assert_eq!(result.ready, 1);
        assert_eq!(result.movePrefix, 1);
        assert_eq!(result.roundBufferPos, prefix.len());
        assert_eq!(
            result.bufferStart,
            round_buffer.as_ptr().wrapping_add(4).cast_mut().cast()
        );

        let blocked_prefix = ZSTDMT_tryGetInputRangeProjection {
            inUseStart: round_buffer.as_ptr().cast(),
            inUseSize: prefix.len(),
            ..wrapped
        };
        assert_eq!(try_get_input_range_with(blocked_prefix).ready, 0);

        let blocked_source = ZSTDMT_tryGetInputRangeProjection {
            roundBufferPos: 8,
            inUseStart: round_buffer.as_ptr().wrapping_add(8).cast(),
            inUseSize: 8,
            ..wrapped
        };
        assert_eq!(try_get_input_range_with(blocked_source).ready, 0);
    }

    #[test]
    fn commit_input_range_wraps_prefix_before_waiting_for_source() {
        let mut round_buffer = [0u8; 32];
        let prefix = [1u8; 4];
        let round_buffer_start = round_buffer.as_mut_ptr();
        let selected_buffer = round_buffer_start.wrapping_add(prefix.len());
        let result = ZSTDMT_tryGetInputRangeResult {
            ready: 1,
            movePrefix: 1,
            bufferStart: selected_buffer.cast(),
            bufferCapacity: 8,
            roundBufferPos: prefix.len(),
        };
        let events = Rc::new(RefCell::new(Vec::new()));
        let wait_events = Rc::clone(&events);
        let move_events = Rc::clone(&events);
        let publish_events = Rc::clone(&events);
        let wait_calls = Rc::new(RefCell::new(Vec::new()));
        let wait_call_records = Rc::clone(&wait_calls);

        let committed = commit_input_range_with(
            result,
            round_buffer_start.cast(),
            prefix.as_ptr().cast(),
            prefix.len(),
            ptr::null_mut(),
            move |_, buffer_start, buffer_capacity| {
                wait_call_records
                    .borrow_mut()
                    .push((buffer_start, buffer_capacity));
                wait_events.borrow_mut().push("wait");
            },
            move |_, destination, source, size, round_buffer_pos| {
                assert_eq!(destination, round_buffer_start.cast());
                assert_eq!(source, prefix.as_ptr().cast());
                assert_eq!(size, prefix.len());
                assert_eq!(round_buffer_pos, prefix.len());
                unsafe {
                    ptr::copy_nonoverlapping(source.cast::<u8>(), destination.cast::<u8>(), size);
                }
                move_events.borrow_mut().push("move");
            },
            move |_, buffer_start, buffer_capacity| {
                assert_eq!(buffer_start, selected_buffer.cast());
                assert_eq!(buffer_capacity, 8);
                publish_events.borrow_mut().push("publish");
            },
        );

        assert_eq!(committed, 1);
        assert_eq!(&*events.borrow(), &["wait", "move", "wait", "publish"]);
        assert_eq!(
            &*wait_calls.borrow(),
            &[
                (round_buffer_start.cast(), prefix.len()),
                (selected_buffer.cast(), 8),
            ]
        );
        assert_eq!(&round_buffer[..prefix.len()], &prefix);
    }

    #[test]
    fn commit_input_range_without_wrap_waits_once_before_publish() {
        let mut round_buffer = [0u8; 32];
        let round_buffer_start = round_buffer.as_mut_ptr();
        let selected_buffer = round_buffer_start.wrapping_add(8);
        let result = ZSTDMT_tryGetInputRangeResult {
            ready: 1,
            bufferStart: selected_buffer.cast(),
            bufferCapacity: 8,
            roundBufferPos: 8,
            ..ZSTDMT_tryGetInputRangeResult::default()
        };
        let events = Rc::new(RefCell::new(Vec::new()));
        let wait_events = Rc::clone(&events);
        let move_called = Rc::new(RefCell::new(false));
        let move_called_by_callback = Rc::clone(&move_called);
        let publish_events = Rc::clone(&events);

        let committed = commit_input_range_with(
            result,
            round_buffer_start.cast(),
            ptr::null(),
            0,
            ptr::null_mut(),
            move |_, buffer_start, buffer_capacity| {
                assert_eq!(buffer_start, selected_buffer.cast());
                assert_eq!(buffer_capacity, 8);
                wait_events.borrow_mut().push("wait");
            },
            move |_, _, _, _, _| {
                *move_called_by_callback.borrow_mut() = true;
            },
            move |_, buffer_start, buffer_capacity| {
                assert_eq!(buffer_start, selected_buffer.cast());
                assert_eq!(buffer_capacity, 8);
                publish_events.borrow_mut().push("publish");
            },
        );

        assert_eq!(committed, 1);
        assert_eq!(&*events.borrow(), &["wait", "publish"]);
        assert!(!*move_called.borrow());
    }

    #[test]
    fn to_flush_now_projects_oldest_slot_and_normalizes_errors() {
        let projection = ZSTDMT_jobProjection {
            consumed: 5,
            cSize: 23,
            srcSize: 17,
            dstFlushed: 9,
            ..ZSTDMT_jobProjection::default()
        };
        let mut seen_slot = None;
        let to_flush = to_flush_now_with(5, 9, 3, |slot| {
            seen_slot = Some(slot);
            projection
        });
        assert_eq!(seen_slot, Some(1));
        assert_eq!(to_flush, 14);

        let error_projection = ZSTDMT_jobProjection {
            consumed: 0,
            cSize: ERROR(ZstdErrorCode::Generic),
            srcSize: 17,
            dstFlushed: 23,
            ..ZSTDMT_jobProjection::default()
        };
        assert_eq!(to_flush_now_with(1, 2, 1, |_| error_projection), 0);

        let mut called = false;
        assert_eq!(
            to_flush_now_with(2, 2, 1, |_| {
                called = true;
                projection
            }),
            0
        );
        assert!(!called);
    }

    fn call_synchronization_point(
        input: &[u8],
        input_pos: usize,
        in_buff: &[u8],
        in_buff_filled: usize,
        target_section_size: usize,
        rsyncable: c_int,
        hit_mask: u64,
    ) -> (usize, c_int) {
        let mut to_load = usize::MAX;
        let mut flush = -1;
        unsafe {
            ZSTDMT_rust_findSynchronizationPoint(
                input.as_ptr().cast(),
                input.len(),
                input_pos,
                target_section_size,
                in_buff.as_ptr().cast(),
                in_buff_filled,
                rsyncable,
                rsync_prime_power(),
                hit_mask,
                &mut to_load,
                &mut flush,
            );
        }
        (to_load, flush)
    }

    fn input_hashes_after_each_byte(
        in_buff: &[u8],
        in_buff_filled: usize,
        input: &[u8],
        input_pos: usize,
    ) -> (u64, Vec<u64>) {
        let mut history = in_buff[in_buff_filled - RSYNC_LENGTH..in_buff_filled].to_vec();
        history.extend_from_slice(&input[input_pos..]);
        let initial = rolling_hash_bytes(&history[..RSYNC_LENGTH]);
        let hashes = (0..input.len() - input_pos)
            .map(|pos| rolling_hash_bytes(&history[pos + 1..pos + 1 + RSYNC_LENGTH]))
            .collect();
        (initial, hashes)
    }

    fn hit_mask_for_index(initial: u64, hashes: &[u64], target: usize) -> u64 {
        let target_hash = hashes[target];
        let prior_matches = |mask: u64| {
            (initial & mask) == mask || hashes[..target].iter().any(|hash| (hash & mask) == mask)
        };
        if target_hash != 0 && !prior_matches(target_hash) {
            return target_hash;
        }
        for mask in 1..=u64::from(u16::MAX) {
            if target_hash & mask == mask && !prior_matches(mask) {
                return mask;
            }
        }
        panic!("could not isolate synchronization hash at position {target}");
    }

    #[test]
    fn synchronization_point_keeps_rsync_disabled() {
        let input = patterned_bytes(20, 3);
        let in_buff = patterned_bytes(64, 5);

        assert_eq!(
            call_synchronization_point(&input, 3, &in_buff, 40, 100, 0, 0),
            (17, 0)
        );
    }

    #[test]
    fn synchronization_point_does_not_scan_a_short_hash_window() {
        let input = patterned_bytes(RSYNC_LENGTH - 1, 7);
        let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 11);

        assert_eq!(
            call_synchronization_point(
                &input,
                0,
                &in_buff,
                0,
                RSYNC_MIN_BLOCK_SIZE + input.len(),
                1,
                0,
            ),
            (input.len(), 0)
        );
    }

    #[test]
    fn synchronization_point_respects_min_block_boundaries() {
        let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 13);
        for &in_buff_filled in &[
            0,
            RSYNC_LENGTH - 1,
            RSYNC_LENGTH,
            RSYNC_MIN_BLOCK_SIZE - 1,
            RSYNC_MIN_BLOCK_SIZE,
        ] {
            let input_len = if in_buff_filled < RSYNC_MIN_BLOCK_SIZE {
                RSYNC_MIN_BLOCK_SIZE - in_buff_filled + 1
            } else {
                1
            };
            let input = patterned_bytes(input_len, in_buff_filled as u8);
            let expected = if in_buff_filled == RSYNC_MIN_BLOCK_SIZE {
                (0, 1)
            } else {
                (input_len, 1)
            };

            assert_eq!(
                call_synchronization_point(
                    &input,
                    0,
                    &in_buff,
                    in_buff_filled,
                    in_buff_filled + input_len,
                    1,
                    0,
                ),
                expected,
                "inBuff.filled={in_buff_filled}"
            );
        }
    }

    #[test]
    fn synchronization_point_reports_initial_first_middle_and_final_hits() {
        let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 17);
        let input = patterned_bytes(96, 19);
        let (initial, hashes) =
            input_hashes_after_each_byte(&in_buff, RSYNC_MIN_BLOCK_SIZE, &input, 0);

        assert_eq!(
            call_synchronization_point(
                &input,
                0,
                &in_buff,
                RSYNC_MIN_BLOCK_SIZE,
                RSYNC_MIN_BLOCK_SIZE + input.len(),
                1,
                0,
            ),
            (0, 1)
        );

        for &(name, index) in &[("first", 0), ("middle", 37), ("final", 95)] {
            let hit_mask = hit_mask_for_index(initial, &hashes, index);
            assert_eq!(
                call_synchronization_point(
                    &input,
                    0,
                    &in_buff,
                    RSYNC_MIN_BLOCK_SIZE,
                    RSYNC_MIN_BLOCK_SIZE + input.len(),
                    1,
                    hit_mask,
                ),
                (index + 1, 1),
                "{name} hit"
            );
        }
    }

    #[test]
    fn synchronization_point_loads_all_input_when_no_hash_hits() {
        let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 23);
        let input = patterned_bytes(96, 29);
        let (initial, hashes) =
            input_hashes_after_each_byte(&in_buff, RSYNC_MIN_BLOCK_SIZE, &input, 0);
        assert_ne!(initial, u64::MAX);
        assert!(hashes.iter().all(|&hash| hash != u64::MAX));

        assert_eq!(
            call_synchronization_point(
                &input,
                0,
                &in_buff,
                RSYNC_MIN_BLOCK_SIZE,
                RSYNC_MIN_BLOCK_SIZE + input.len(),
                1,
                u64::MAX,
            ),
            (input.len(), 0)
        );
    }

    #[test]
    fn synchronization_point_preserves_nonzero_input_position() {
        let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 31);
        let prefix = patterned_bytes(7, 37);
        let payload = patterned_bytes(64, 41);
        let mut input = prefix;
        input.extend_from_slice(&payload);
        let (initial, hashes) =
            input_hashes_after_each_byte(&in_buff, RSYNC_MIN_BLOCK_SIZE, &input, 7);
        let hit_mask = hit_mask_for_index(initial, &hashes, 23);

        assert_eq!(
            call_synchronization_point(
                &input,
                7,
                &in_buff,
                RSYNC_MIN_BLOCK_SIZE,
                RSYNC_MIN_BLOCK_SIZE + payload.len(),
                1,
                hit_mask,
            ),
            (24, 1)
        );
    }

    #[test]
    fn synchronization_point_handles_a_window_spanning_the_buffer_boundary() {
        let in_buff = patterned_bytes(RSYNC_MIN_BLOCK_SIZE, 43);
        let input = [0xe1, 0x7a];
        let in_buff_filled = RSYNC_MIN_BLOCK_SIZE - 1;
        let mut history = in_buff[in_buff_filled - RSYNC_LENGTH..in_buff_filled].to_vec();
        history.extend_from_slice(&input);
        let hit_mask = rolling_hash_bytes(&history[2..2 + RSYNC_LENGTH]);

        assert_eq!(
            call_synchronization_point(
                &input,
                0,
                &in_buff,
                in_buff_filled,
                in_buff_filled + input.len(),
                1,
                hit_mask,
            ),
            (input.len(), 1)
        );
    }

    #[test]
    fn raw_seq_buffer_conversion_uses_whole_element_capacity() {
        let mut sequences = [ZstdMtRawSeq::default(); 3];
        let element_size = mem::size_of::<ZstdMtRawSeq>();
        let buffer = ZstdMtBuffer {
            start: sequences.as_mut_ptr().cast(),
            capacity: element_size * sequences.len() + element_size - 1,
        };

        let seq = ZSTDMT_rust_bufferToSeq(buffer);
        assert_eq!(seq.seq, sequences.as_mut_ptr());
        assert_eq!(seq.pos, 0);
        assert_eq!(seq.posInSequence, 0);
        assert_eq!(seq.size, 0);
        assert_eq!(seq.capacity, sequences.len());

        let roundtrip = ZSTDMT_rust_seqToBuffer(seq);
        assert_eq!(roundtrip.start, buffer.start);
        assert_eq!(roundtrip.capacity, element_size * sequences.len());
    }

    #[test]
    fn raw_seq_to_buffer_preserves_pointer_and_size_t_multiplication() {
        let seq = ZstdMtRawSeqStore {
            seq: ptr::null_mut(),
            pos: 4,
            posInSequence: 5,
            size: 6,
            capacity: usize::MAX,
        };
        let buffer = ZSTDMT_rust_seqToBuffer(seq);
        assert!(buffer.start.is_null());
        assert_eq!(
            buffer.capacity,
            usize::MAX.wrapping_mul(mem::size_of::<ZstdMtRawSeq>())
        );

        let empty = ZSTDMT_rust_bufferToSeq(ZstdMtBuffer {
            start: ptr::null_mut(),
            capacity: mem::size_of::<ZstdMtRawSeq>() - 1,
        });
        assert!(empty.seq.is_null());
        assert_eq!(empty.capacity, 0);
        assert_eq!(empty.pos, 0);
        assert_eq!(empty.posInSequence, 0);
        assert_eq!(empty.size, 0);
    }

    #[test]
    fn overlapped_rejects_null_ranges() {
        let bytes = [0u8; 8];
        let start = bytes.as_ptr().cast::<c_void>();

        assert_eq!(ZSTDMT_rust_isOverlapped(ptr::null(), 4, start, 4), 0);
        assert_eq!(ZSTDMT_rust_isOverlapped(start, 4, ptr::null(), 4), 0);
    }

    #[test]
    fn overlapped_rejects_empty_ranges() {
        let bytes = [0u8; 8];
        let start = bytes.as_ptr().cast::<c_void>();

        assert_eq!(ZSTDMT_rust_isOverlapped(start, 0, start, 4), 0);
        assert_eq!(ZSTDMT_rust_isOverlapped(start, 4, start, 0), 0);
    }

    #[test]
    fn overlapped_uses_half_open_range_boundaries() {
        let bytes = [0u8; 16];
        let start = bytes.as_ptr();
        let middle = start.wrapping_add(4);

        assert_eq!(
            ZSTDMT_rust_isOverlapped(start.cast(), 4, middle.cast(), 4),
            0
        );
        assert_eq!(
            ZSTDMT_rust_isOverlapped(middle.cast(), 4, start.cast(), 4),
            0
        );
    }

    #[test]
    fn overlapped_detects_contained_ranges() {
        let bytes = [0u8; 16];
        let start = bytes.as_ptr();
        let inner = start.wrapping_add(4);

        assert_eq!(
            ZSTDMT_rust_isOverlapped(start.cast(), 12, inner.cast(), 4),
            1
        );
        assert_eq!(
            ZSTDMT_rust_isOverlapped(inner.cast(), 4, start.cast(), 12),
            1
        );
    }

    #[test]
    fn overlapped_rejects_disjoint_ranges() {
        let bytes = [0u8; 16];
        let start = bytes.as_ptr();
        let after = start.wrapping_add(8);

        assert_eq!(
            ZSTDMT_rust_isOverlapped(start.cast(), 4, after.cast(), 4),
            0
        );
        assert_eq!(
            ZSTDMT_rust_isOverlapped(after.cast(), 4, start.cast(), 4),
            0
        );
    }

    #[test]
    fn overlap_window_checks_external_dictionary_and_prefix() {
        let bytes = [0u8; 32];
        let base = bytes.as_ptr();
        let next_src = base.wrapping_add(16);
        let dict_base = base.wrapping_add(16);

        assert_eq!(
            ZSTDMT_rust_doesOverlapWindow(
                base.wrapping_add(20).cast(),
                4,
                next_src.cast(),
                base.cast(),
                dict_base.cast(),
                8,
                4,
            ),
            1
        );
        assert_eq!(
            ZSTDMT_rust_doesOverlapWindow(
                base.wrapping_add(12).cast(),
                4,
                next_src.cast(),
                base.cast(),
                dict_base.cast(),
                8,
                4,
            ),
            1
        );
        assert_eq!(
            ZSTDMT_rust_doesOverlapWindow(
                base.cast(),
                4,
                next_src.cast(),
                base.cast(),
                dict_base.cast(),
                8,
                4,
            ),
            0
        );
    }

    #[test]
    fn direct_overlap_window_abi_matches_scalar_policy() {
        let bytes = [0u8; 32];
        let base = bytes.as_ptr();
        let window = ZstdMtWindow {
            nextSrc: base.wrapping_add(16),
            base,
            dictBase: base.wrapping_add(16),
            dictLimit: 8,
            lowLimit: 4,
            nbOverflowCorrections: 0,
        };
        let buffer = ZstdMtBuffer {
            start: base.wrapping_add(20) as *mut c_void,
            capacity: 4,
        };
        assert_eq!(ZSTDMT_doesOverlapWindow(buffer, window), 1);
        assert_eq!(
            ZSTDMT_doesOverlapWindow(
                ZstdMtBuffer {
                    start: base as *mut u8 as *mut c_void,
                    capacity: 4,
                },
                window,
            ),
            0
        );
    }

    #[test]
    fn buffer_pool_reuses_and_resizes_buffers() {
        let pool = unsafe { ZSTDMT_rust_buffer_pool_create(2, DEFAULT_MEM) };
        assert!(!pool.is_null());

        let first = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
        assert!(!first.start.is_null());
        assert_eq!(first.capacity, 64 << 10);
        unsafe { ZSTDMT_rust_buffer_pool_release(pool, first) };

        let reused = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
        assert_eq!(reused.start, first.start);
        assert_eq!(reused.capacity, first.capacity);
        unsafe { ZSTDMT_rust_buffer_pool_release(pool, reused) };

        unsafe { ZSTDMT_rust_buffer_pool_set_size(pool, 128 << 10) };
        let resized = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
        assert_eq!(resized.capacity, 128 << 10);
        unsafe {
            ZSTDMT_rust_buffer_pool_release(pool, resized);
            ZSTDMT_rust_buffer_pool_free(pool);
        }
    }

    #[test]
    fn zero_size_pool_does_not_allocate_sequence_storage() {
        let pool = unsafe { ZSTDMT_rust_buffer_pool_create(1, DEFAULT_MEM) };
        assert!(!pool.is_null());
        unsafe { ZSTDMT_rust_buffer_pool_set_size(pool, 0) };
        let buffer = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
        assert!(buffer.start.is_null());
        assert_eq!(buffer.capacity, 0);
        unsafe { ZSTDMT_rust_buffer_pool_free(pool) };
    }

    #[test]
    fn expansion_preserves_requested_buffer_size() {
        let pool = unsafe { ZSTDMT_rust_buffer_pool_create(1, DEFAULT_MEM) };
        assert!(!pool.is_null());
        unsafe { ZSTDMT_rust_buffer_pool_set_size(pool, 96 << 10) };
        let expanded = unsafe { ZSTDMT_rust_buffer_pool_expand(pool, 3) };
        assert!(!expanded.is_null());
        let buffer = unsafe { ZSTDMT_rust_buffer_pool_get(expanded) };
        assert_eq!(buffer.capacity, 96 << 10);
        unsafe {
            ZSTDMT_rust_buffer_pool_release(expanded, buffer);
            ZSTDMT_rust_buffer_pool_free(expanded);
        }
    }

    #[test]
    fn resize_preserves_existing_buffer_contents() {
        let pool = unsafe { ZSTDMT_rust_buffer_pool_create(1, DEFAULT_MEM) };
        assert!(!pool.is_null());
        let buffer = unsafe { ZSTDMT_rust_buffer_pool_get(pool) };
        assert!(!buffer.start.is_null());
        let sample = b"multithreaded-buffer-pool";
        unsafe {
            ptr::copy_nonoverlapping(sample.as_ptr(), buffer.start.cast(), sample.len());
            ZSTDMT_rust_buffer_pool_set_size(pool, buffer.capacity + 1);
        }
        let resized = unsafe { ZSTDMT_rust_buffer_pool_resize(pool, buffer) };
        assert_eq!(resized.capacity, buffer.capacity + 1);
        let contents =
            unsafe { std::slice::from_raw_parts(resized.start.cast::<u8>(), sample.len()) };
        assert_eq!(contents, sample);
        unsafe {
            ZSTDMT_rust_buffer_pool_release(pool, resized);
            ZSTDMT_rust_buffer_pool_free(pool);
        }
    }

    #[test]
    fn job_table_count_matches_c_power_of_two_contract() {
        assert_eq!(rounded_job_count(0), None);
        assert_eq!(rounded_job_count(1), Some(2));
        assert_eq!(rounded_job_count(3), Some(4));
        assert_eq!(rounded_job_count(4), Some(8));
        assert_eq!(rounded_job_count(255), Some(256));
        assert_eq!(rounded_job_count(256), Some(512));
    }

    #[test]
    fn job_table_creation_cleans_up_after_sync_init_failure() {
        let _lock = JOB_TABLE_TEST_LOCK.lock().unwrap();
        JOB_TABLE_INIT_CALLS.store(0, Ordering::Relaxed);
        JOB_TABLE_DESTROY_CALLS.store(0, Ordering::Relaxed);
        JOB_TABLE_FAIL_INIT.store(true, Ordering::Relaxed);

        let mut requested_jobs = 1;
        let job_table = unsafe {
            ZSTDMT_rust_createJobsTable(
                &mut requested_jobs,
                1,
                DEFAULT_MEM,
                probe_job_table_init,
                probe_job_table_destroy,
            )
        };

        assert!(job_table.is_null());
        assert_eq!(requested_jobs, 2);
        assert_eq!(JOB_TABLE_INIT_CALLS.load(Ordering::Relaxed), 1);
        assert_eq!(JOB_TABLE_DESTROY_CALLS.load(Ordering::Relaxed), 1);

        JOB_TABLE_FAIL_INIT.store(false, Ordering::Relaxed);
    }

    #[test]
    fn job_table_expansion_preserves_mask_and_sync_lifecycle() {
        let _lock = JOB_TABLE_TEST_LOCK.lock().unwrap();
        JOB_TABLE_INIT_CALLS.store(0, Ordering::Relaxed);
        JOB_TABLE_DESTROY_CALLS.store(0, Ordering::Relaxed);
        JOB_TABLE_FAIL_INIT.store(false, Ordering::Relaxed);

        let mut requested_jobs = 1;
        let initial_table =
            unsafe { ZSTDMT_rust_job_table_create(&mut requested_jobs, 1, DEFAULT_MEM) };
        assert!(!initial_table.is_null());
        assert_eq!(requested_jobs, 2);

        let mut job_table = initial_table;
        let mut job_id_mask = 1;
        assert_eq!(
            unsafe {
                ZSTDMT_rust_expandJobsTable(
                    &mut job_table,
                    &mut job_id_mask,
                    2,
                    1,
                    DEFAULT_MEM,
                    probe_job_table_init,
                    probe_job_table_destroy,
                )
            },
            0
        );
        assert!(!job_table.is_null());
        assert_eq!(job_id_mask, 7);
        assert_eq!(JOB_TABLE_INIT_CALLS.load(Ordering::Relaxed), 1);
        assert_eq!(JOB_TABLE_DESTROY_CALLS.load(Ordering::Relaxed), 1);

        let expanded_table = job_table;
        assert_eq!(
            unsafe {
                ZSTDMT_rust_expandJobsTable(
                    &mut job_table,
                    &mut job_id_mask,
                    2,
                    1,
                    DEFAULT_MEM,
                    probe_job_table_init,
                    probe_job_table_destroy,
                )
            },
            0
        );
        assert_eq!(job_table, expanded_table);
        assert_eq!(job_id_mask, 7);
        assert_eq!(JOB_TABLE_INIT_CALLS.load(Ordering::Relaxed), 1);
        assert_eq!(JOB_TABLE_DESTROY_CALLS.load(Ordering::Relaxed), 1);

        unsafe { ZSTDMT_rust_job_table_free(job_table, 8, 1, DEFAULT_MEM) };
    }

    #[test]
    fn job_table_expansion_cleans_up_after_sync_init_failure() {
        let _lock = JOB_TABLE_TEST_LOCK.lock().unwrap();
        JOB_TABLE_INIT_CALLS.store(0, Ordering::Relaxed);
        JOB_TABLE_DESTROY_CALLS.store(0, Ordering::Relaxed);
        JOB_TABLE_FAIL_INIT.store(true, Ordering::Relaxed);

        let mut requested_jobs = 1;
        let initial_table =
            unsafe { ZSTDMT_rust_job_table_create(&mut requested_jobs, 1, DEFAULT_MEM) };
        assert!(!initial_table.is_null());

        let mut job_table = initial_table;
        let mut job_id_mask = 1;
        assert_eq!(
            unsafe {
                ZSTDMT_rust_expandJobsTable(
                    &mut job_table,
                    &mut job_id_mask,
                    2,
                    1,
                    DEFAULT_MEM,
                    probe_job_table_init,
                    probe_job_table_destroy,
                )
            },
            ERROR(ZstdErrorCode::MemoryAllocation)
        );
        assert!(job_table.is_null());
        assert_eq!(job_id_mask, 0);
        assert_eq!(JOB_TABLE_INIT_CALLS.load(Ordering::Relaxed), 1);
        assert_eq!(JOB_TABLE_DESTROY_CALLS.load(Ordering::Relaxed), 2);

        JOB_TABLE_FAIL_INIT.store(false, Ordering::Relaxed);
    }

    #[test]
    fn target_job_log_preserves_ldm_and_non_ldm_policy() {
        assert_eq!(
            compute_target_job_log(18, 25, ZSTD_FAST, ZSTD_PS_DISABLE),
            20
        );
        assert_eq!(
            compute_target_job_log(30, 25, ZSTD_FAST, ZSTD_PS_DISABLE),
            ZSTDMT_JOBLOG_MAX
        );
        assert_eq!(
            compute_target_job_log(10, 25, ZSTD_FAST, ZSTD_PS_ENABLE),
            28
        );
        assert_eq!(
            compute_target_job_log(30, 25, ZSTD_BTULTRA2, ZSTD_PS_ENABLE),
            27
        );
        assert_eq!(
            compute_target_job_log(10, 30, ZSTD_BTULTRA2, ZSTD_PS_ENABLE),
            ZSTDMT_JOBLOG_MAX
        );
    }

    #[test]
    fn compression_job_parameters_disable_worker_owned_features() {
        assert_eq!(
            prepare_compression_job_parameters(0, 1, ZSTD_PS_ENABLE, 4),
            ZSTDMT_compressionJobParameters {
                checksumFlag: 1,
                ldmEnable: ZSTD_PS_DISABLE,
                nbWorkers: 0,
            }
        );
    }

    #[test]
    fn compression_job_parameters_clear_checksum_after_first_job() {
        assert_eq!(
            prepare_compression_job_parameters(3, 1, ZSTD_PS_ENABLE, 4),
            ZSTDMT_compressionJobParameters {
                checksumFlag: 0,
                ldmEnable: ZSTD_PS_DISABLE,
                nbWorkers: 0,
            }
        );
        assert_eq!(
            prepare_compression_job_parameters(9, 0, ZSTD_PS_DISABLE, 0),
            ZSTDMT_compressionJobParameters {
                checksumFlag: 0,
                ldmEnable: ZSTD_PS_DISABLE,
                nbWorkers: 0,
            }
        );
    }

    #[test]
    fn overlap_log_defaults_follow_strategy_groups() {
        assert_eq!(overlap_log_default(ZSTD_FAST), 6);
        assert_eq!(overlap_log_default(ZSTD_DFAST), 6);
        assert_eq!(overlap_log_default(ZSTD_GREEDY), 6);
        assert_eq!(overlap_log_default(ZSTD_LAZY), 6);
        assert_eq!(overlap_log_default(ZSTD_LAZY2), 7);
        assert_eq!(overlap_log_default(ZSTD_BTLAZY2), 7);
        assert_eq!(overlap_log_default(ZSTD_BTOPT), 8);
        assert_eq!(overlap_log_default(ZSTD_BTULTRA), 8);
        assert_eq!(overlap_log_default(ZSTD_BTULTRA2), 9);
        assert_eq!(overlap_log_default(0), 6);

        assert_eq!(overlap_log(0, ZSTD_BTULTRA2), 9);
        assert_eq!(overlap_log(0, ZSTD_FAST), 6);
        assert_eq!(overlap_log(5, ZSTD_BTULTRA2), 5);
        assert_eq!(overlap_log(9, ZSTD_FAST), 9);
    }

    #[test]
    fn overlap_size_uses_window_or_target_job_log_as_expected() {
        assert_eq!(
            compute_overlap_size(20, 25, ZSTD_FAST, 0, ZSTD_PS_DISABLE),
            1usize << 17
        );
        assert_eq!(
            compute_overlap_size(20, 25, ZSTD_BTULTRA2, 0, ZSTD_PS_DISABLE),
            1usize << 20
        );
        assert_eq!(
            compute_overlap_size(20, 25, ZSTD_FAST, 1, ZSTD_PS_DISABLE),
            0
        );
        assert_eq!(
            compute_overlap_size(30, 25, ZSTD_FAST, 0, ZSTD_PS_ENABLE),
            1usize << 23
        );
        assert_eq!(
            compute_overlap_size(30, 25, ZSTD_BTULTRA2, 0, ZSTD_PS_ENABLE),
            1usize << 25
        );
    }

    #[test]
    fn cctx_pool_size_addition_handles_zero_components() {
        assert_eq!(sizeof_cctx_pool(0, 0), 0);
        assert_eq!(sizeof_cctx_pool(37, 0), 37);
        assert_eq!(sizeof_cctx_pool(0, 53), 53);
    }

    #[test]
    fn cctx_pool_size_addition_preserves_ordinary_sum() {
        assert_eq!(sizeof_cctx_pool(128, 4096), 4224);
    }

    #[test]
    fn cctx_pool_size_addition_wraps_like_c_size_t() {
        assert_eq!(sizeof_cctx_pool(usize::MAX, 1), 0);
        assert_eq!(sizeof_cctx_pool(usize::MAX - 7, 11), 3);
    }

    #[test]
    fn buffer_pool_size_addition_handles_zero_components() {
        assert_eq!(sizeof_buffer_pool(0, 0), 0);
        assert_eq!(sizeof_buffer_pool(37, 0), 37);
        assert_eq!(sizeof_buffer_pool(0, 53), 53);
    }

    #[test]
    fn buffer_pool_size_addition_preserves_ordinary_sum() {
        assert_eq!(sizeof_buffer_pool(128, 4096), 4224);
    }

    #[test]
    fn buffer_pool_size_addition_wraps_like_c_size_t() {
        assert_eq!(sizeof_buffer_pool(usize::MAX, 1), 0);
        assert_eq!(sizeof_buffer_pool(usize::MAX - 7, 11), 3);
    }
}