zstd-rs/rust/src/zstd_decompress_block.rs

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]

//! Compressed-block decoding.
//!
//! The C decoder context deliberately remains opaque.  The companion C shim
//! extracts only the pointers and scalar values this translation needs into
//! [`ZSTD_rustBlockCtx`], so optional C context fields cannot silently change
//! this Rust module's ABI.  Literal parsing, FSE table construction, sequence
//! decoding, and block execution live here.

use crate::bitstream::{
    BIT_DStream_t, BIT_endOfDStream, BIT_initDStream, BIT_readBits, BIT_readBitsFast,
    BIT_reloadDStream,
};
use crate::common::{
    DEFAULT_MAX_OFF, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_FSE_LOG, MAX_LL, MAX_ML,
    MAX_OFF, MIN_CBLOCK_SIZE, MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, ML_BITS,
    ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_REP_NUM,
};
use crate::entropy_common::FSE_readNCount;
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
#[cfg(not(feature = "huf-force-decompress-x2"))]
use crate::huf_decompress::HUF_decompress1X1_DCtx_wksp;
#[cfg(feature = "huf-force-decompress-x2")]
use crate::huf_decompress::HUF_decompress1X_DCtx_wksp;
use crate::huf_decompress::{
    HUF_decompress1X_usingDTable, HUF_decompress4X_hufOnly_wksp, HUF_decompress4X_usingDTable,
};
use crate::mem::{MEM_32bits, MEM_64bits, MEM_readLE16, MEM_readLE24, U32};
use crate::zstd_decompress::ZSTD_DCtx;
use std::cmp::min;
use std::ffi::c_void;
use std::mem::MaybeUninit;
use std::os::raw::{c_int, c_short, c_uint};
use std::ptr;
use std::sync::OnceLock;

const ZSTD_BLOCKSIZE_MAX: usize = 128 << 10;
const ZSTD_BLOCK_HEADER_SIZE: usize = 3;
const WILDCOPY_OVERLENGTH: usize = 32;
const HUF_FLAGS_BMI2: c_int = 1 << 0;
const HUF_FLAGS_DISABLE_ASM: c_int = 1 << 4;
const SET_BASIC: c_int = 0;
const SET_RLE: c_int = 1;
const SET_COMPRESSED: c_int = 2;
const SET_REPEAT: c_int = 3;
const NOT_STREAMING: c_int = 0;
const ZSTD_NOT_IN_DST: c_int = 0;
const ZSTD_IN_DST: c_int = 1;
const ZSTD_SPLIT: c_int = 2;
const LONG_NB_SEQ: usize = 0x7f00;
const LL_FSE_LOG: u32 = 9;
const OFF_FSE_LOG: u32 = 8;
const ML_FSE_LOG: u32 = 9;
const ZSTD_HUFFDTABLE_CAPACITY_LOG: usize = 12;
const HUF_DTABLE_SIZE: usize = 1 + (1 << ZSTD_HUFFDTABLE_CAPACITY_LOG);
const ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32: usize = 157;
const STREAM_ACCUMULATOR_MIN_32: usize = 25;
const STREAM_ACCUMULATOR_MIN_64: usize = 57;
const SEQUENCE_DECODER_RUNTIME: c_int = 0;
const SEQUENCE_DECODER_FORCE_SHORT: c_int = 1;
const SEQUENCE_DECODER_FORCE_LONG: c_int = 2;
const LONG_OFFSET_ADDITIONAL_BITS: u8 = 22;
const LONG_OFFSET_HISTORY_THRESHOLD: usize = 1 << 24;
const LONG_OFFSET_MIN_SHARE_32: u32 = 20;
const LONG_OFFSET_MIN_SHARE_64: u32 = 7;

const LL_BASE: [u32; MAX_LL + 1] = [
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, 48, 64,
    0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000,
];
const OF_BASE: [u32; MAX_OFF + 1] = [
    0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
    0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD,
    0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD,
];
const OF_BITS: [u8; MAX_OFF + 1] = [
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
    26, 27, 28, 29, 30, 31,
];
const ML_BASE: [u32; MAX_ML + 1] = [
    3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
    28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203,
    0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003,
];

/// `ZSTD_seqSymbol` from `zstd_decompress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct ZSTD_seqSymbol {
    next_state: u16,
    nb_additional_bits: u8,
    nb_bits: u8,
    base_value: u32,
}

/// The first entry of a sequence table is overlaid as this C header.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
struct ZSTD_seqSymbol_header {
    fast_mode: u32,
    table_log: u32,
}

#[repr(C)]
struct ZSTD_entropyDTables_t {
    ll_table: [ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)],
    of_table: [ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)],
    ml_table: [ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)],
    huf_table: [u32; HUF_DTABLE_SIZE],
    rep: [u32; ZSTD_REP_NUM],
    workspace: [u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32],
}

/// C-owned leaves of `ZSTD_DCtx_s` used by this translation unit.
///
/// Every pointer is produced by `zstd_decompress_block.c` under the active C
/// configuration.  This avoids assuming offsets for optional context members
/// such as `DYNAMIC_BMI2`, fuzzing bounds, and tracing state.
#[repr(C)]
pub struct ZSTD_rustBlockCtx {
    llt_ptr: *mut *const ZSTD_seqSymbol,
    mlt_ptr: *mut *const ZSTD_seqSymbol,
    oft_ptr: *mut *const ZSTD_seqSymbol,
    huf_ptr: *mut *const u32,
    entropy: *mut ZSTD_entropyDTables_t,
    workspace: *mut u32,
    workspace_size: usize,
    previous_dst_end: *mut *const u8,
    prefix_start: *mut *const u8,
    virtual_start: *mut *const u8,
    dict_end: *mut *const u8,
    block_size_max: usize,
    is_frame_decompression: *mut c_int,
    lit_entropy: *mut u32,
    fse_entropy: *mut u32,
    bmi2: c_int,
    ddict_is_cold: *mut c_int,
    disable_huf_asm: c_int,
    lit_ptr: *mut *const u8,
    lit_size: *mut usize,
    rle_size: *mut usize,
    lit_buffer: *mut *mut u8,
    lit_buffer_end: *mut *const u8,
    lit_buffer_location: *mut c_int,
    lit_extra_buffer: *mut u8,
    lit_extra_buffer_size: usize,
    sequence_decoder_mode: c_int,
}

unsafe extern "C" {
    fn ZSTD_rust_block_context_init(out: *mut ZSTD_rustBlockCtx, dctx: *mut ZSTD_DCtx);
}

#[inline]
unsafe fn block_context(dctx: *mut ZSTD_DCtx) -> ZSTD_rustBlockCtx {
    let mut ctx = MaybeUninit::<ZSTD_rustBlockCtx>::uninit();
    unsafe { ZSTD_rust_block_context_init(ctx.as_mut_ptr(), dctx) };
    unsafe { ctx.assume_init() }
}

#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct ZSTD_rustSeq {
    lit_length: usize,
    match_length: usize,
    offset: usize,
}

#[repr(C)]
struct block_properties_t {
    block_type: c_int,
    last_block: u32,
    orig_size: u32,
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct ZSTD_offsetInfo {
    long_offset_share: u32,
    max_nb_additional_bits: u8,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SequenceDecoder {
    Short,
    Long,
}

#[derive(Clone, Copy)]
struct ZSTD_fseState {
    state: usize,
    table: *const ZSTD_seqSymbol,
}

struct seq_state_t {
    dstream: BIT_DStream_t,
    state_ll: ZSTD_fseState,
    state_off: ZSTD_fseState,
    state_ml: ZSTD_fseState,
    prev_offset: [usize; ZSTD_REP_NUM],
}

#[inline]
unsafe fn entropy(ctx: *mut ZSTD_rustBlockCtx) -> *mut ZSTD_entropyDTables_t {
    unsafe { (*ctx).entropy }
}

#[inline]
unsafe fn address_distance(end: *const u8, start: *const u8) -> usize {
    (end as usize).wrapping_sub(start as usize)
}

#[inline]
unsafe fn ptr_add(ptr: *mut u8, amount: usize) -> *mut u8 {
    if ptr.is_null() {
        debug_assert_eq!(amount, 0);
        ptr
    } else {
        unsafe { ptr.add(amount) }
    }
}

#[inline]
unsafe fn const_ptr_add(ptr: *const u8, amount: usize) -> *const u8 {
    if ptr.is_null() {
        debug_assert_eq!(amount, 0);
        ptr
    } else {
        unsafe { ptr.add(amount) }
    }
}

#[inline]
unsafe fn seq_header(table: *const ZSTD_seqSymbol) -> ZSTD_seqSymbol_header {
    unsafe { table.cast::<ZSTD_seqSymbol_header>().read_unaligned() }
}

#[inline]
unsafe fn set_seq_header(table: *mut ZSTD_seqSymbol, header: ZSTD_seqSymbol_header) {
    unsafe {
        table
            .cast::<ZSTD_seqSymbol_header>()
            .write_unaligned(header)
    };
}

#[inline]
fn stream_accumulator_min() -> usize {
    if MEM_32bits() {
        STREAM_ACCUMULATOR_MIN_32
    } else {
        STREAM_ACCUMULATOR_MIN_64
    }
}

/// Port of the C decoder's `ZSTD_getOffsetInfo()` policy helper.
///
/// `long_offset_share` is scaled to the full offset-table log, so the caller
/// can compare tables with different table logs using the same thresholds.
/// The table is valid when this helper is reached from sequence-header
/// decoding; the defensive checks keep the helper harmless in focused tests
/// and preserve the C zero-sequence behavior.
#[inline]
unsafe fn get_offset_info(off_table: *const ZSTD_seqSymbol, nb_seq: c_int) -> ZSTD_offsetInfo {
    if nb_seq == 0 || off_table.is_null() {
        return ZSTD_offsetInfo::default();
    }
    let table_log = unsafe { seq_header(off_table).table_log };
    if table_log > OFF_FSE_LOG {
        return ZSTD_offsetInfo::default();
    }
    let table_size = 1usize << table_log;
    let table = unsafe { off_table.add(1) };
    let mut info = ZSTD_offsetInfo::default();
    for index in 0..table_size {
        let additional_bits = unsafe { (*table.add(index)).nb_additional_bits };
        info.max_nb_additional_bits = info.max_nb_additional_bits.max(additional_bits);
        if additional_bits > LONG_OFFSET_ADDITIONAL_BITS {
            info.long_offset_share = info.long_offset_share.wrapping_add(1);
        }
    }
    info.long_offset_share <<= OFF_FSE_LOG - table_log;
    info
}

#[inline]
fn choose_sequence_decoder(
    mode: c_int,
    ddict_is_cold: c_int,
    potential_long_offsets: bool,
    total_history_size: usize,
    nb_seq: c_int,
    offset_info: ZSTD_offsetInfo,
) -> SequenceDecoder {
    match mode {
        SEQUENCE_DECODER_FORCE_SHORT => return SequenceDecoder::Short,
        SEQUENCE_DECODER_FORCE_LONG => return SequenceDecoder::Long,
        SEQUENCE_DECODER_RUNTIME => {}
        _ => {}
    }

    let mut use_prefetch_decoder = ddict_is_cold != 0;
    if !use_prefetch_decoder
        && (potential_long_offsets
            || (total_history_size > LONG_OFFSET_HISTORY_THRESHOLD && nb_seq > 8))
    {
        let minimum_share = if MEM_32bits() {
            LONG_OFFSET_MIN_SHARE_32
        } else {
            LONG_OFFSET_MIN_SHARE_64
        };
        use_prefetch_decoder = offset_info.long_offset_share >= minimum_share;
    }
    if use_prefetch_decoder {
        SequenceDecoder::Long
    } else {
        SequenceDecoder::Short
    }
}

#[inline]
unsafe fn copy_bytes(dst: *mut u8, src: *const u8, len: usize) {
    if len != 0 {
        unsafe { ptr::copy(src, dst, len) };
    }
}

#[inline]
unsafe fn fill_bytes(dst: *mut u8, value: u8, len: usize) {
    if len != 0 {
        unsafe { ptr::write_bytes(dst, value, len) };
    }
}

/// Copy a match using forward byte semantics, which is required for repeated
/// short-offset matches (unlike `memmove`, which would not expand overlap).
#[inline]
unsafe fn copy_match(mut dst: *mut u8, mut src: *const u8, len: usize) {
    for _ in 0..len {
        unsafe { dst.write(src.read()) };
        dst = unsafe { dst.add(1) };
        src = unsafe { src.add(1) };
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_getcBlockSize(
    src: *const c_void,
    src_size: usize,
    bp_ptr: *mut c_void,
) -> usize {
    if src_size < ZSTD_BLOCK_HEADER_SIZE {
        return ERROR(ZstdErrorCode::SrcSizeWrong);
    }
    let header = unsafe { MEM_readLE24(src) };
    let bp = bp_ptr.cast::<block_properties_t>();
    unsafe {
        (*bp).last_block = header & 1;
        (*bp).block_type = ((header >> 1) & 3) as c_int;
        (*bp).orig_size = header >> 3;
    }
    if unsafe { (*bp).block_type } == 1 {
        return 1;
    }
    if unsafe { (*bp).block_type } == 3 {
        return ERROR(ZstdErrorCode::CorruptionDetected);
    }
    (header >> 3) as usize
}

#[inline]
unsafe fn block_size_max(ctx: *const ZSTD_rustBlockCtx) -> usize {
    let value = unsafe {
        if *(*ctx).is_frame_decompression != 0 {
            (*ctx).block_size_max
        } else {
            ZSTD_BLOCKSIZE_MAX
        }
    };
    debug_assert!(value <= ZSTD_BLOCKSIZE_MAX);
    value
}

#[inline]
unsafe fn total_history_size(
    ctx: *const ZSTD_rustBlockCtx,
    dst: *mut u8,
    dst_capacity: usize,
) -> usize {
    let block_size = min(dst_capacity, unsafe { block_size_max(ctx) });
    let history_end = unsafe { ptr_add(dst, block_size) };
    unsafe { address_distance(history_end.cast_const(), *(*ctx).virtual_start) }
}

unsafe fn allocate_literals_buffer(
    ctx: *mut ZSTD_rustBlockCtx,
    dst: *mut u8,
    dst_capacity: usize,
    lit_size: usize,
    streaming: c_int,
    expected_write_size: usize,
    split_immediately: bool,
) {
    let block_max = unsafe { block_size_max(ctx) };
    debug_assert!(lit_size <= block_max);
    debug_assert!(*unsafe { (*ctx).is_frame_decompression } != 0 || streaming == NOT_STREAMING);
    if streaming == NOT_STREAMING
        && dst_capacity
            > block_max
                .saturating_add(WILDCOPY_OVERLENGTH)
                .saturating_add(lit_size)
                .saturating_add(WILDCOPY_OVERLENGTH)
    {
        let buffer = unsafe { ptr_add(dst, block_max + WILDCOPY_OVERLENGTH) };
        unsafe {
            *(*ctx).lit_buffer = buffer;
            *(*ctx).lit_buffer_end = buffer.add(lit_size);
            *(*ctx).lit_buffer_location = ZSTD_IN_DST;
        }
    } else if lit_size <= unsafe { (*ctx).lit_extra_buffer_size } {
        let buffer = unsafe { (*ctx).lit_extra_buffer };
        unsafe {
            *(*ctx).lit_buffer = buffer;
            *(*ctx).lit_buffer_end = buffer.add(lit_size);
            *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST;
        }
    } else {
        let extra = unsafe { (*ctx).lit_extra_buffer_size };
        debug_assert!(block_max > extra);
        let buffer = if split_immediately {
            unsafe {
                ptr_add(
                    dst,
                    expected_write_size - lit_size + extra - WILDCOPY_OVERLENGTH,
                )
            }
        } else {
            unsafe { ptr_add(dst, expected_write_size - lit_size) }
        };
        unsafe {
            *(*ctx).lit_buffer = buffer;
            *(*ctx).lit_buffer_end = if split_immediately {
                buffer.add(lit_size - extra)
            } else {
                ptr_add(dst, expected_write_size).cast_const()
            };
            *(*ctx).lit_buffer_location = ZSTD_SPLIT;
        }
    }
}

unsafe fn decode_literals_block(
    ctx: *mut ZSTD_rustBlockCtx,
    src: *const u8,
    src_size: usize,
    dst: *mut u8,
    dst_capacity: usize,
    streaming: c_int,
) -> usize {
    if src_size < MIN_CBLOCK_SIZE {
        return ERROR(ZstdErrorCode::CorruptionDetected);
    }
    let lit_type = unsafe { *src & 3 } as c_int;
    let block_max = unsafe { block_size_max(ctx) };
    match lit_type {
        SET_REPEAT | SET_COMPRESSED => {
            if lit_type == SET_REPEAT && unsafe { *(*ctx).lit_entropy } == 0 {
                return ERROR(ZstdErrorCode::DictionaryCorrupted);
            }
            if src_size < 5 {
                return ERROR(ZstdErrorCode::CorruptionDetected);
            }
            let lhl_code = unsafe { (*src >> 2) & 3 };
            let header = unsafe { crate::mem::MEM_readLE32(src.cast()) };
            let (header_size, lit_size, lit_c_size, single_stream) = match lhl_code {
                0 | 1 => (
                    3usize,
                    ((header >> 4) & 0x3ff) as usize,
                    ((header >> 14) & 0x3ff) as usize,
                    lhl_code == 0,
                ),
                2 => (
                    4usize,
                    ((header >> 4) & 0x3fff) as usize,
                    (header >> 18) as usize,
                    false,
                ),
                _ => (
                    5usize,
                    ((header >> 4) & 0x3ffff) as usize,
                    ((header >> 22) as usize).wrapping_add((unsafe { *src.add(4) } as usize) << 10),
                    false,
                ),
            };
            if (lit_size != 0 && dst.is_null())
                || lit_size > block_max
                || (!single_stream && lit_size < MIN_LITERALS_FOR_4_STREAMS)
                || header_size.checked_add(lit_c_size).is_none()
                || header_size + lit_c_size > src_size
                || min(block_max, dst_capacity) < lit_size
            {
                return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
                {
                    ERROR(ZstdErrorCode::DstSizeTooSmall)
                } else {
                    ERROR(ZstdErrorCode::CorruptionDetected)
                };
            }
            unsafe {
                allocate_literals_buffer(
                    ctx,
                    dst,
                    dst_capacity,
                    lit_size,
                    streaming,
                    min(block_max, dst_capacity),
                    false,
                );
            }
            let flags = (if unsafe { (*ctx).bmi2 } != 0 {
                HUF_FLAGS_BMI2
            } else {
                0
            }) | (if unsafe { (*ctx).disable_huf_asm } != 0 {
                HUF_FLAGS_DISABLE_ASM
            } else {
                0
            });
            let lit_buffer = unsafe { *(*ctx).lit_buffer };
            let huf_result = if lit_type == SET_REPEAT {
                if single_stream {
                    unsafe {
                        HUF_decompress1X_usingDTable(
                            lit_buffer.cast(),
                            lit_size,
                            src.add(header_size).cast(),
                            lit_c_size,
                            *(*ctx).huf_ptr,
                            flags,
                        )
                    }
                } else {
                    unsafe {
                        HUF_decompress4X_usingDTable(
                            lit_buffer.cast(),
                            lit_size,
                            src.add(header_size).cast(),
                            lit_c_size,
                            *(*ctx).huf_ptr,
                            flags,
                        )
                    }
                }
            } else if single_stream {
                #[cfg(feature = "huf-force-decompress-x2")]
                {
                    unsafe {
                        HUF_decompress1X_DCtx_wksp(
                            (*entropy(ctx)).huf_table.as_mut_ptr(),
                            lit_buffer.cast(),
                            lit_size,
                            src.add(header_size).cast(),
                            lit_c_size,
                            (*ctx).workspace.cast(),
                            (*ctx).workspace_size,
                            flags,
                        )
                    }
                }
                #[cfg(not(feature = "huf-force-decompress-x2"))]
                {
                    unsafe {
                        HUF_decompress1X1_DCtx_wksp(
                            (*entropy(ctx)).huf_table.as_mut_ptr(),
                            lit_buffer.cast(),
                            lit_size,
                            src.add(header_size).cast(),
                            lit_c_size,
                            (*ctx).workspace.cast(),
                            (*ctx).workspace_size,
                            flags,
                        )
                    }
                }
            } else {
                unsafe {
                    HUF_decompress4X_hufOnly_wksp(
                        (*entropy(ctx)).huf_table.as_mut_ptr(),
                        lit_buffer.cast(),
                        lit_size,
                        src.add(header_size).cast(),
                        lit_c_size,
                        (*ctx).workspace.cast(),
                        (*ctx).workspace_size,
                        flags,
                    )
                }
            };
            if ERR_isError(huf_result) {
                return ERROR(ZstdErrorCode::CorruptionDetected);
            }
            if unsafe { *(*ctx).lit_buffer_location } == ZSTD_SPLIT {
                let extra = unsafe { (*ctx).lit_extra_buffer_size };
                unsafe {
                    copy_bytes(
                        (*ctx).lit_extra_buffer,
                        (*(*ctx).lit_buffer_end).sub(extra),
                        extra,
                    );
                    ptr::copy(
                        *(*ctx).lit_buffer,
                        (*(*ctx).lit_buffer).add(extra - WILDCOPY_OVERLENGTH),
                        lit_size - extra,
                    );
                    *(*ctx).lit_buffer = (*(*ctx).lit_buffer).add(extra - WILDCOPY_OVERLENGTH);
                    *(*ctx).lit_buffer_end = (*(*ctx).lit_buffer_end).sub(WILDCOPY_OVERLENGTH);
                }
            }
            unsafe {
                *(*ctx).lit_ptr = *(*ctx).lit_buffer;
                *(*ctx).lit_size = lit_size;
                *(*ctx).lit_entropy = 1;
                if lit_type == SET_COMPRESSED {
                    *(*ctx).huf_ptr = (*entropy(ctx)).huf_table.as_ptr();
                }
            }
            header_size + lit_c_size
        }
        SET_BASIC => {
            let lhl_code = unsafe { (*src >> 2) & 3 };
            let (header_size, lit_size) = match lhl_code {
                0 | 2 => (1usize, (unsafe { *src } >> 3) as usize),
                1 => (2usize, (unsafe { MEM_readLE16(src.cast()) } >> 4) as usize),
                _ => {
                    if src_size < 3 {
                        return ERROR(ZstdErrorCode::CorruptionDetected);
                    }
                    (3usize, (unsafe { MEM_readLE24(src.cast()) } >> 4) as usize)
                }
            };
            if (lit_size != 0 && dst.is_null())
                || lit_size > block_max
                || min(block_max, dst_capacity) < lit_size
            {
                return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
                {
                    ERROR(ZstdErrorCode::DstSizeTooSmall)
                } else {
                    ERROR(ZstdErrorCode::CorruptionDetected)
                };
            }
            unsafe {
                allocate_literals_buffer(
                    ctx,
                    dst,
                    dst_capacity,
                    lit_size,
                    streaming,
                    min(block_max, dst_capacity),
                    true,
                );
            }
            if header_size
                .checked_add(lit_size)
                .and_then(|size| size.checked_add(WILDCOPY_OVERLENGTH))
                .is_none_or(|size| size > src_size)
            {
                if header_size
                    .checked_add(lit_size)
                    .is_none_or(|size| size > src_size)
                {
                    return ERROR(ZstdErrorCode::CorruptionDetected);
                }
                unsafe {
                    if *(*ctx).lit_buffer_location == ZSTD_SPLIT {
                        let extra = (*ctx).lit_extra_buffer_size;
                        copy_bytes(*(*ctx).lit_buffer, src.add(header_size), lit_size - extra);
                        copy_bytes(
                            (*ctx).lit_extra_buffer,
                            src.add(header_size + lit_size - extra),
                            extra,
                        );
                    } else {
                        copy_bytes(*(*ctx).lit_buffer, src.add(header_size), lit_size);
                    }
                    *(*ctx).lit_ptr = *(*ctx).lit_buffer;
                    *(*ctx).lit_size = lit_size;
                }
                return header_size + lit_size;
            }
            unsafe {
                *(*ctx).lit_ptr = src.add(header_size);
                *(*ctx).lit_size = lit_size;
                *(*ctx).lit_buffer_end = src.add(header_size + lit_size);
                *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST;
            }
            header_size + lit_size
        }
        SET_RLE => {
            let lhl_code = unsafe { (*src >> 2) & 3 };
            let (header_size, lit_size) = match lhl_code {
                0 | 2 => (1usize, (unsafe { *src } >> 3) as usize),
                1 => {
                    if src_size < 3 {
                        return ERROR(ZstdErrorCode::CorruptionDetected);
                    }
                    (2usize, (unsafe { MEM_readLE16(src.cast()) } >> 4) as usize)
                }
                _ => {
                    if src_size < 4 {
                        return ERROR(ZstdErrorCode::CorruptionDetected);
                    }
                    (3usize, (unsafe { MEM_readLE24(src.cast()) } >> 4) as usize)
                }
            };
            if (lit_size != 0 && dst.is_null())
                || lit_size > block_max
                || min(block_max, dst_capacity) < lit_size
            {
                return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
                {
                    ERROR(ZstdErrorCode::DstSizeTooSmall)
                } else {
                    ERROR(ZstdErrorCode::CorruptionDetected)
                };
            }
            unsafe {
                allocate_literals_buffer(
                    ctx,
                    dst,
                    dst_capacity,
                    lit_size,
                    streaming,
                    min(block_max, dst_capacity),
                    true,
                );
                let value = *src.add(header_size);
                if *(*ctx).lit_buffer_location == ZSTD_SPLIT {
                    let extra = (*ctx).lit_extra_buffer_size;
                    fill_bytes(*(*ctx).lit_buffer, value, lit_size - extra);
                    fill_bytes((*ctx).lit_extra_buffer, value, extra);
                } else {
                    fill_bytes(*(*ctx).lit_buffer, value, lit_size);
                }
                *(*ctx).lit_ptr = *(*ctx).lit_buffer;
                *(*ctx).lit_size = lit_size;
            }
            header_size + 1
        }
        _ => ERROR(ZstdErrorCode::CorruptionDetected),
    }
}

unsafe fn build_seq_table_rle(table: *mut ZSTD_seqSymbol, base_value: u32, nb_add_bits: u8) {
    unsafe {
        set_seq_header(
            table,
            ZSTD_seqSymbol_header {
                fast_mode: 0,
                table_log: 0,
            },
        );
        *table.add(1) = ZSTD_seqSymbol {
            next_state: 0,
            nb_additional_bits: nb_add_bits,
            nb_bits: 0,
            base_value,
        };
    }
}

unsafe fn build_fse_table_body(
    table: *mut ZSTD_seqSymbol,
    normalized_counter: *const i16,
    max_symbol_value: u32,
    base_value: *const u32,
    nb_additional_bits: *const u8,
    table_log: u32,
    workspace: *mut u32,
    workspace_size: usize,
) {
    debug_assert!(max_symbol_value as usize <= MAX_ML);
    debug_assert!(table_log as usize <= MAX_FSE_LOG);
    debug_assert!(
        workspace_size >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 * std::mem::size_of::<u32>()
    );
    let table_decode = unsafe { table.add(1) };
    let table_size = 1usize << table_log;
    let symbol_next = workspace.cast::<u16>();
    let spread = unsafe { symbol_next.add(MAX_ML + 1).cast::<u8>() };
    let mut high_threshold = table_size - 1;
    let mut fast_mode = 1u32;
    let large_limit = 1i16 << (table_log - 1);
    for symbol in 0..=max_symbol_value as usize {
        let count = unsafe { *normalized_counter.add(symbol) };
        if count == -1 {
            unsafe {
                (*table_decode.add(high_threshold)).base_value = symbol as u32;
                *symbol_next.add(symbol) = 1;
            }
            high_threshold = high_threshold.wrapping_sub(1);
        } else {
            if count >= large_limit {
                fast_mode = 0;
            }
            debug_assert!(count >= 0);
            unsafe { *symbol_next.add(symbol) = count as u16 };
        }
    }
    unsafe {
        set_seq_header(
            table,
            ZSTD_seqSymbol_header {
                fast_mode,
                table_log,
            },
        );
    }
    let table_mask = table_size - 1;
    let step = (table_size >> 1) + (table_size >> 3) + 3;
    if high_threshold == table_size - 1 {
        let mut pos = 0usize;
        for symbol in 0..=max_symbol_value as usize {
            let count = unsafe { *normalized_counter.add(symbol) };
            debug_assert!(count >= 0);
            for index in 0..count as usize {
                unsafe { *spread.add(pos + index) = symbol as u8 };
            }
            pos += count as usize;
        }
        let mut position = 0usize;
        for symbol_index in 0..table_size {
            unsafe { (*table_decode.add(position)).base_value = *spread.add(symbol_index) as u32 };
            position = (position + step) & table_mask;
        }
        debug_assert_eq!(position, 0);
    } else {
        let mut position = 0usize;
        for symbol in 0..=max_symbol_value as usize {
            let count = unsafe { *normalized_counter.add(symbol) };
            for _ in 0..count.max(0) as usize {
                unsafe { (*table_decode.add(position)).base_value = symbol as u32 };
                position = (position + step) & table_mask;
                while position > high_threshold {
                    position = (position + step) & table_mask;
                }
            }
        }
        debug_assert_eq!(position, 0);
    }
    for index in 0..table_size {
        let symbol = unsafe { (*table_decode.add(index)).base_value as usize };
        let next_state = unsafe { *symbol_next.add(symbol) } as u32;
        unsafe { *symbol_next.add(symbol) = next_state.wrapping_add(1) as u16 };
        let nb_bits = table_log - crate::bits::ZSTD_highbit32(next_state);
        unsafe {
            let entry = &mut *table_decode.add(index);
            entry.nb_bits = nb_bits as u8;
            entry.next_state = ((next_state << nb_bits) - table_size as u32) as u16;
            entry.nb_additional_bits = *nb_additional_bits.add(symbol);
            entry.base_value = *base_value.add(symbol);
        }
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_buildFSETable_body(
    table: *mut ZSTD_seqSymbol,
    normalized_counter: *const c_short,
    max_symbol_value: c_uint,
    base_value: *const U32,
    nb_additional_bits: *const u8,
    table_log: c_uint,
    workspace: *mut c_void,
    workspace_size: usize,
) {
    unsafe {
        build_fse_table_body(
            table,
            normalized_counter,
            max_symbol_value,
            base_value,
            nb_additional_bits,
            table_log,
            workspace.cast(),
            workspace_size,
        );
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_buildFSETable(
    table: *mut ZSTD_seqSymbol,
    normalized_counter: *const c_short,
    max_symbol_value: c_uint,
    base_value: *const U32,
    nb_additional_bits: *const u8,
    table_log: c_uint,
    workspace: *mut c_void,
    workspace_size: usize,
    _bmi2: c_int,
) {
    unsafe {
        build_fse_table_body(
            table,
            normalized_counter,
            max_symbol_value,
            base_value,
            nb_additional_bits,
            table_log,
            workspace.cast(),
            workspace_size,
        );
    }
}

fn build_default_table<const N: usize>(
    normalized: &[i16],
    base: &[u32],
    bits: &[u8],
    max_symbol_value: u32,
    table_log: u32,
) -> [ZSTD_seqSymbol; N] {
    let mut table = [ZSTD_seqSymbol::default(); N];
    let mut workspace = [0u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
    unsafe {
        build_fse_table_body(
            table.as_mut_ptr(),
            normalized.as_ptr(),
            max_symbol_value,
            base.as_ptr(),
            bits.as_ptr(),
            table_log,
            workspace.as_mut_ptr(),
            std::mem::size_of_val(&workspace),
        );
    }
    table
}

static LL_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)]> = OnceLock::new();
static OF_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)]> = OnceLock::new();
static ML_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)]> = OnceLock::new();

#[inline]
fn ll_default_table() -> *const ZSTD_seqSymbol {
    LL_DEFAULT_TABLE
        .get_or_init(|| {
            build_default_table(
                &LL_DEFAULT_NORM,
                &LL_BASE,
                &LL_BITS,
                MAX_LL as u32,
                LL_DEFAULT_NORM_LOG,
            )
        })
        .as_ptr()
}

#[inline]
fn of_default_table() -> *const ZSTD_seqSymbol {
    OF_DEFAULT_TABLE
        .get_or_init(|| {
            build_default_table(
                &OF_DEFAULT_NORM,
                &OF_BASE,
                &OF_BITS,
                DEFAULT_MAX_OFF as u32,
                OF_DEFAULT_NORM_LOG,
            )
        })
        .as_ptr()
}

#[inline]
fn ml_default_table() -> *const ZSTD_seqSymbol {
    ML_DEFAULT_TABLE
        .get_or_init(|| {
            build_default_table(
                &ML_DEFAULT_NORM,
                &ML_BASE,
                &ML_BITS,
                MAX_ML as u32,
                ML_DEFAULT_NORM_LOG,
            )
        })
        .as_ptr()
}

unsafe fn build_seq_table(
    table_space: *mut ZSTD_seqSymbol,
    table_ptr: *mut *const ZSTD_seqSymbol,
    table_type: c_int,
    mut max_symbol: u32,
    max_log: u32,
    src: *const u8,
    src_size: usize,
    base: &[u32],
    bits: &[u8],
    default_table: *const ZSTD_seqSymbol,
    repeat_table_available: u32,
    _ddict_is_cold: c_int,
    _nb_seq: c_int,
    workspace: *mut u32,
    workspace_size: usize,
    bmi2: c_int,
) -> usize {
    match table_type {
        SET_RLE => {
            if src_size == 0 {
                return ERROR(ZstdErrorCode::SrcSizeWrong);
            }
            let symbol = unsafe { *src } as usize;
            if symbol > max_symbol as usize {
                return ERROR(ZstdErrorCode::CorruptionDetected);
            }
            unsafe { build_seq_table_rle(table_space, base[symbol], bits[symbol]) };
            unsafe { *table_ptr = table_space };
            1
        }
        SET_BASIC => {
            unsafe { *table_ptr = default_table };
            0
        }
        SET_REPEAT => {
            if repeat_table_available == 0 {
                ERROR(ZstdErrorCode::CorruptionDetected)
            } else {
                0
            }
        }
        SET_COMPRESSED => {
            let mut table_log = 0u32;
            let mut norm = [0i16; MAX_ML + 1];
            let header_size = unsafe {
                FSE_readNCount(
                    norm.as_mut_ptr(),
                    &mut max_symbol,
                    &mut table_log,
                    src.cast(),
                    src_size,
                )
            };
            if ERR_isError(header_size) || table_log > max_log {
                return ERROR(ZstdErrorCode::CorruptionDetected);
            }
            unsafe {
                build_fse_table_body(
                    table_space,
                    norm.as_ptr(),
                    max_symbol,
                    base.as_ptr(),
                    bits.as_ptr(),
                    table_log,
                    workspace,
                    workspace_size,
                );
                *table_ptr = table_space;
            }
            let _ = bmi2;
            header_size
        }
        _ => ERROR(ZstdErrorCode::Generic),
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decodeSeqHeaders(
    ctx: *mut ZSTD_rustBlockCtx,
    nb_seq_ptr: *mut c_int,
    src: *const c_void,
    src_size: usize,
) -> usize {
    if src_size < MIN_SEQUENCES_SIZE {
        return ERROR(ZstdErrorCode::SrcSizeWrong);
    }
    let start = src.cast::<u8>();
    let end = unsafe { start.add(src_size) };
    let mut ip = start;
    let mut nb_seq = unsafe { *ip } as usize;
    ip = unsafe { ip.add(1) };
    if nb_seq > 0x7f {
        if nb_seq == 0xff {
            if (ip as usize).wrapping_add(2) > end as usize {
                return ERROR(ZstdErrorCode::SrcSizeWrong);
            }
            nb_seq = unsafe { MEM_readLE16(ip.cast()) as usize } + LONG_NB_SEQ;
            ip = unsafe { ip.add(2) };
        } else {
            if ip >= end {
                return ERROR(ZstdErrorCode::SrcSizeWrong);
            }
            nb_seq = ((nb_seq - 0x80) << 8) + unsafe { *ip } as usize;
            ip = unsafe { ip.add(1) };
        }
    }
    unsafe { *nb_seq_ptr = nb_seq as c_int };
    if nb_seq == 0 {
        return if ip == end {
            unsafe { ip.offset_from(start) as usize }
        } else {
            ERROR(ZstdErrorCode::CorruptionDetected)
        };
    }
    if ip >= end || unsafe { *ip & 3 } != 0 {
        return if ip >= end {
            ERROR(ZstdErrorCode::SrcSizeWrong)
        } else {
            ERROR(ZstdErrorCode::CorruptionDetected)
        };
    }
    let descriptor = unsafe { *ip };
    ip = unsafe { ip.add(1) };
    let ll_type = (descriptor >> 6) as c_int;
    let of_type = ((descriptor >> 4) & 3) as c_int;
    let ml_type = ((descriptor >> 2) & 3) as c_int;
    let entropy = unsafe { entropy(ctx) };
    let repeat = unsafe { *(*ctx).fse_entropy };
    let cold = unsafe { *(*ctx).ddict_is_cold };
    let ll_size = unsafe {
        build_seq_table(
            (*entropy).ll_table.as_mut_ptr(),
            (*ctx).llt_ptr,
            ll_type,
            MAX_LL as u32,
            LL_FSE_LOG,
            ip,
            address_distance(end, ip),
            &LL_BASE,
            &LL_BITS,
            ll_default_table(),
            repeat,
            cold,
            nb_seq as c_int,
            (*ctx).workspace,
            (*ctx).workspace_size,
            (*ctx).bmi2,
        )
    };
    if ERR_isError(ll_size) || ll_size > unsafe { address_distance(end, ip) } {
        return ERROR(ZstdErrorCode::CorruptionDetected);
    }
    ip = unsafe { ip.add(ll_size) };
    let of_size = unsafe {
        build_seq_table(
            (*entropy).of_table.as_mut_ptr(),
            (*ctx).oft_ptr,
            of_type,
            MAX_OFF as u32,
            OFF_FSE_LOG,
            ip,
            address_distance(end, ip),
            &OF_BASE,
            &OF_BITS,
            of_default_table(),
            repeat,
            cold,
            nb_seq as c_int,
            (*ctx).workspace,
            (*ctx).workspace_size,
            (*ctx).bmi2,
        )
    };
    if ERR_isError(of_size) || of_size > unsafe { address_distance(end, ip) } {
        return ERROR(ZstdErrorCode::CorruptionDetected);
    }
    ip = unsafe { ip.add(of_size) };
    let ml_size = unsafe {
        build_seq_table(
            (*entropy).ml_table.as_mut_ptr(),
            (*ctx).mlt_ptr,
            ml_type,
            MAX_ML as u32,
            ML_FSE_LOG,
            ip,
            address_distance(end, ip),
            &ML_BASE,
            &ML_BITS,
            ml_default_table(),
            repeat,
            cold,
            nb_seq as c_int,
            (*ctx).workspace,
            (*ctx).workspace_size,
            (*ctx).bmi2,
        )
    };
    if ERR_isError(ml_size) || ml_size > unsafe { address_distance(end, ip) } {
        return ERROR(ZstdErrorCode::CorruptionDetected);
    }
    ip = unsafe { ip.add(ml_size) };
    unsafe { ip.offset_from(start) as usize }
}

#[inline]
unsafe fn init_fse_state(
    state: &mut ZSTD_fseState,
    dstream: *mut BIT_DStream_t,
    table: *const ZSTD_seqSymbol,
) -> Result<(), usize> {
    if table.is_null() {
        return Err(ERROR(ZstdErrorCode::CorruptionDetected));
    }
    let header = unsafe { seq_header(table) };
    if header.table_log > MAX_FSE_LOG as u32 {
        return Err(ERROR(ZstdErrorCode::CorruptionDetected));
    }
    state.state = unsafe { BIT_readBits(dstream, header.table_log) };
    let _ = unsafe { BIT_reloadDStream(dstream) };
    state.table = unsafe { table.add(1) };
    Ok(())
}

#[inline]
unsafe fn update_fse_state(
    state: &mut ZSTD_fseState,
    dstream: *mut BIT_DStream_t,
    next_state: u16,
    nb_bits: u8,
) {
    let low_bits = unsafe { BIT_readBits(dstream, nb_bits as u32) };
    state.state = next_state as usize + low_bits;
}

unsafe fn decode_sequence(
    state: &mut seq_state_t,
    long_offsets: bool,
    is_last_sequence: bool,
) -> ZSTD_rustSeq {
    let ll_info = unsafe { state.state_ll.table.add(state.state_ll.state).read() };
    let ml_info = unsafe { state.state_ml.table.add(state.state_ml.state).read() };
    let of_info = unsafe { state.state_off.table.add(state.state_off.state).read() };
    let ll_bits = ll_info.nb_additional_bits as u32;
    let ml_bits = ml_info.nb_additional_bits as u32;
    let of_bits = of_info.nb_additional_bits as u32;
    let total_bits = ll_bits + ml_bits + of_bits;
    let mut sequence = ZSTD_rustSeq {
        lit_length: ll_info.base_value as usize,
        match_length: ml_info.base_value as usize,
        offset: 0,
    };

    let offset = if of_bits > 1 {
        let value = if MEM_32bits() && long_offsets && of_bits >= 25 {
            let upper = unsafe { BIT_readBitsFast(&mut state.dstream, of_bits - 5) } << 5;
            let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
            upper + unsafe { BIT_readBitsFast(&mut state.dstream, 5) }
        } else {
            let value = unsafe { BIT_readBitsFast(&mut state.dstream, of_bits) };
            if MEM_32bits() {
                let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
            }
            value
        };
        let value = (of_info.base_value as usize).wrapping_add(value);
        state.prev_offset[2] = state.prev_offset[1];
        state.prev_offset[1] = state.prev_offset[0];
        state.prev_offset[0] = value;
        value
    } else {
        let ll_zero = ll_info.base_value == 0;
        if of_bits == 0 {
            let value = state.prev_offset[usize::from(ll_zero)];
            state.prev_offset[1] = state.prev_offset[usize::from(!ll_zero)];
            state.prev_offset[0] = value;
            value
        } else {
            let code = (of_info.base_value as usize)
                .wrapping_add(usize::from(ll_zero))
                .wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, 1) });
            let mut value = if code == 3 {
                state.prev_offset[0].wrapping_sub(1)
            } else if code < ZSTD_REP_NUM {
                state.prev_offset[code]
            } else {
                usize::MAX
            };
            value = value.wrapping_sub(usize::from(value == 0));
            if code != 1 {
                state.prev_offset[2] = state.prev_offset[1];
            }
            state.prev_offset[1] = state.prev_offset[0];
            state.prev_offset[0] = value;
            value
        }
    };
    sequence.offset = offset;
    if ml_bits != 0 {
        sequence.match_length = sequence
            .match_length
            .wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, ml_bits) });
    }
    if MEM_32bits() && ml_bits + ll_bits >= 20 {
        let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
    }
    if MEM_64bits() && total_bits >= 30 {
        let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
    }
    if ll_bits != 0 {
        sequence.lit_length = sequence
            .lit_length
            .wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, ll_bits) });
    }
    if MEM_32bits() {
        let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
    }
    if !is_last_sequence {
        unsafe {
            update_fse_state(
                &mut state.state_ll,
                &mut state.dstream,
                ll_info.next_state,
                ll_info.nb_bits,
            );
            update_fse_state(
                &mut state.state_ml,
                &mut state.dstream,
                ml_info.next_state,
                ml_info.nb_bits,
            );
        }
        if MEM_32bits() {
            let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
        }
        unsafe {
            update_fse_state(
                &mut state.state_off,
                &mut state.dstream,
                of_info.next_state,
                of_info.nb_bits,
            );
        }
        let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
    }
    sequence
}

unsafe fn exec_sequence(
    op: *mut u8,
    oend: *mut u8,
    sequence: ZSTD_rustSeq,
    lit_ptr: &mut *const u8,
    lit_limit: *const u8,
    prefix_start: *const u8,
    virtual_start: *const u8,
    dict_end: *const u8,
    split_literals: bool,
) -> usize {
    let sequence_length = match sequence.lit_length.checked_add(sequence.match_length) {
        Some(length) => length,
        None => return ERROR(ZstdErrorCode::DstSizeTooSmall),
    };
    let output_capacity = unsafe { address_distance(oend.cast_const(), op.cast_const()) };
    if sequence_length > output_capacity {
        return ERROR(ZstdErrorCode::DstSizeTooSmall);
    }
    let literal_available = unsafe { address_distance(lit_limit, *lit_ptr) };
    if sequence.lit_length > literal_available {
        return ERROR(ZstdErrorCode::CorruptionDetected);
    }
    let lit_end = unsafe { ptr_add(op, sequence.lit_length) };
    if split_literals
        && (op as usize) > (*lit_ptr as usize)
        && (op as usize) < (*lit_ptr as usize).wrapping_add(sequence.lit_length)
    {
        return ERROR(ZstdErrorCode::DstSizeTooSmall);
    }
    unsafe { copy_bytes(op, *lit_ptr, sequence.lit_length) };
    *lit_ptr = unsafe { const_ptr_add(*lit_ptr, sequence.lit_length) };
    if sequence.offset == 0 {
        return ERROR(ZstdErrorCode::CorruptionDetected);
    }
    let prefix_history = unsafe { address_distance(lit_end.cast_const(), prefix_start) };
    let mut output = lit_end;
    let mut match_length = sequence.match_length;
    if sequence.offset > prefix_history {
        let virtual_history = unsafe { address_distance(lit_end.cast_const(), virtual_start) };
        if sequence.offset > virtual_history || dict_end.is_null() {
            return ERROR(ZstdErrorCode::CorruptionDetected);
        }
        let before_prefix = sequence.offset - prefix_history;
        let match_ptr = unsafe { dict_end.sub(before_prefix) };
        let dict_available = unsafe { address_distance(dict_end, match_ptr) };
        let first_length = min(match_length, dict_available);
        unsafe { copy_bytes(output, match_ptr, first_length) };
        output = unsafe { output.add(first_length) };
        match_length -= first_length;
        if match_length != 0 {
            if prefix_start.is_null() {
                return ERROR(ZstdErrorCode::CorruptionDetected);
            }
            unsafe { copy_match(output, prefix_start, match_length) };
        }
    } else {
        let match_ptr = unsafe { lit_end.sub(sequence.offset) };
        if match_ptr.is_null() {
            return ERROR(ZstdErrorCode::CorruptionDetected);
        }
        unsafe { copy_match(output, match_ptr, match_length) };
    }
    sequence_length
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceEnd(
    op: *mut u8,
    oend: *mut u8,
    sequence: ZSTD_rustSeq,
    lit_ptr: *mut *const u8,
    lit_limit: *const u8,
    prefix_start: *const u8,
    virtual_start: *const u8,
    dict_end: *const u8,
) -> usize {
    unsafe {
        exec_sequence(
            op,
            oend,
            sequence,
            &mut *lit_ptr,
            lit_limit,
            prefix_start,
            virtual_start,
            dict_end,
            false,
        )
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceEndSplitLitBuffer(
    op: *mut u8,
    oend: *mut u8,
    _oend_w: *const u8,
    sequence: ZSTD_rustSeq,
    lit_ptr: *mut *const u8,
    lit_limit: *const u8,
    prefix_start: *const u8,
    virtual_start: *const u8,
    dict_end: *const u8,
) -> usize {
    unsafe {
        exec_sequence(
            op,
            oend,
            sequence,
            &mut *lit_ptr,
            lit_limit,
            prefix_start,
            virtual_start,
            dict_end,
            true,
        )
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequence(
    op: *mut u8,
    oend: *mut u8,
    sequence: ZSTD_rustSeq,
    lit_ptr: *mut *const u8,
    lit_limit: *const u8,
    prefix_start: *const u8,
    virtual_start: *const u8,
    dict_end: *const u8,
) -> usize {
    unsafe {
        exec_sequence(
            op,
            oend,
            sequence,
            &mut *lit_ptr,
            lit_limit,
            prefix_start,
            virtual_start,
            dict_end,
            false,
        )
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceSplitLitBuffer(
    op: *mut u8,
    oend: *mut u8,
    _oend_w: *const u8,
    sequence: ZSTD_rustSeq,
    lit_ptr: *mut *const u8,
    lit_limit: *const u8,
    prefix_start: *const u8,
    virtual_start: *const u8,
    dict_end: *const u8,
) -> usize {
    unsafe {
        exec_sequence(
            op,
            oend,
            sequence,
            &mut *lit_ptr,
            lit_limit,
            prefix_start,
            virtual_start,
            dict_end,
            true,
        )
    }
}

unsafe fn decompress_sequences(
    ctx: *mut ZSTD_rustBlockCtx,
    dst: *mut u8,
    dst_capacity: usize,
    sequence_start: *const u8,
    sequence_size: usize,
    nb_seq: c_int,
    long_offsets: bool,
    use_prefetch_decoder: bool,
) -> usize {
    if nb_seq < 0 {
        return ERROR(ZstdErrorCode::CorruptionDetected);
    }
    let mut location = unsafe { *(*ctx).lit_buffer_location };
    let mut op = dst;
    let mut oend = if location == ZSTD_NOT_IN_DST || location == ZSTD_SPLIT {
        unsafe { ptr_add(dst, dst_capacity) }
    } else {
        unsafe { *(*ctx).lit_buffer }
    };
    let mut lit_ptr = unsafe { *(*ctx).lit_ptr };
    let mut lit_limit = if location == ZSTD_SPLIT {
        unsafe { *(*ctx).lit_buffer_end }
    } else {
        unsafe { const_ptr_add(lit_ptr, *(*ctx).lit_size) }
    };
    let prefix_start = unsafe { *(*ctx).prefix_start };
    let virtual_start = unsafe { *(*ctx).virtual_start };
    let dict_end = unsafe { *(*ctx).dict_end };
    if nb_seq != 0 {
        if dst.is_null() {
            return ERROR(ZstdErrorCode::DstSizeTooSmall);
        }
        let mut state = seq_state_t {
            dstream: unsafe { MaybeUninit::zeroed().assume_init() },
            state_ll: ZSTD_fseState {
                state: 0,
                table: ptr::null(),
            },
            state_off: ZSTD_fseState {
                state: 0,
                table: ptr::null(),
            },
            state_ml: ZSTD_fseState {
                state: 0,
                table: ptr::null(),
            },
            prev_offset: unsafe { (*entropy(ctx)).rep.map(|value| value as usize) },
        };
        unsafe { *(*ctx).fse_entropy = 1 };
        let init =
            unsafe { BIT_initDStream(&mut state.dstream, sequence_start.cast(), sequence_size) };
        if ERR_isError(init) {
            return ERROR(ZstdErrorCode::CorruptionDetected);
        }
        if let Err(error) =
            unsafe { init_fse_state(&mut state.state_ll, &mut state.dstream, *(*ctx).llt_ptr) }
        {
            return error;
        }
        if let Err(error) =
            unsafe { init_fse_state(&mut state.state_off, &mut state.dstream, *(*ctx).oft_ptr) }
        {
            return error;
        }
        if let Err(error) =
            unsafe { init_fse_state(&mut state.state_ml, &mut state.dstream, *(*ctx).mlt_ptr) }
        {
            return error;
        }
        let mut prefetch_pos = 0usize;
        for remaining in (1..=nb_seq as usize).rev() {
            let mut sequence = unsafe { decode_sequence(&mut state, long_offsets, remaining == 1) };
            if use_prefetch_decoder {
                prefetch_pos =
                    unsafe { ZSTD_prefetchMatch(prefetch_pos, sequence, prefix_start, dict_end) };
            }
            if location == ZSTD_SPLIT
                && sequence.lit_length > unsafe { address_distance(lit_limit, lit_ptr) }
            {
                let leftover = unsafe { address_distance(lit_limit, lit_ptr) };
                if leftover > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
                    return ERROR(ZstdErrorCode::DstSizeTooSmall);
                }
                unsafe { copy_bytes(op, lit_ptr, leftover) };
                op = unsafe { op.add(leftover) };
                sequence.lit_length -= leftover;
                lit_ptr = unsafe { (*ctx).lit_extra_buffer.cast_const() };
                lit_limit = unsafe {
                    const_ptr_add(
                        (*ctx).lit_extra_buffer.cast_const(),
                        (*ctx).lit_extra_buffer_size,
                    )
                };
                location = ZSTD_NOT_IN_DST;
                unsafe { *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST };
            }
            let decoded = unsafe {
                exec_sequence(
                    op,
                    oend,
                    sequence,
                    &mut lit_ptr,
                    lit_limit,
                    prefix_start,
                    virtual_start,
                    dict_end,
                    location == ZSTD_SPLIT,
                )
            };
            if ERR_isError(decoded) {
                return decoded;
            }
            op = unsafe { op.add(decoded) };
        }
        if unsafe { BIT_endOfDStream(&state.dstream) } == 0 {
            return ERROR(ZstdErrorCode::CorruptionDetected);
        }
        unsafe { (*entropy(ctx)).rep = state.prev_offset.map(|value| value as u32) };
    }
    if location == ZSTD_SPLIT {
        let first_size = unsafe { address_distance(lit_limit, lit_ptr) };
        if first_size > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
            return ERROR(ZstdErrorCode::DstSizeTooSmall);
        }
        unsafe { copy_bytes(op, lit_ptr, first_size) };
        op = unsafe { op.add(first_size) };
        lit_ptr = unsafe { (*ctx).lit_extra_buffer.cast_const() };
        lit_limit = unsafe {
            const_ptr_add(
                (*ctx).lit_extra_buffer.cast_const(),
                (*ctx).lit_extra_buffer_size,
            )
        };
        unsafe { *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST };
        oend = unsafe { ptr_add(dst, dst_capacity) };
    }
    let last_size = unsafe { address_distance(lit_limit, lit_ptr) };
    if last_size > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
        return ERROR(ZstdErrorCode::DstSizeTooSmall);
    }
    unsafe { copy_bytes(op, lit_ptr, last_size) };
    op = unsafe { op.add(last_size) };
    unsafe { address_distance(op.cast_const(), dst.cast_const()) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_prefetchMatch(
    prefetch_pos: usize,
    sequence: ZSTD_rustSeq,
    _prefix_start: *const u8,
    _dict_end: *const u8,
) -> usize {
    prefetch_pos
        .wrapping_add(sequence.lit_length)
        .wrapping_add(sequence.match_length)
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decodeLiteralsBlock_wrapper(
    ctx: *mut ZSTD_rustBlockCtx,
    src: *const c_void,
    src_size: usize,
    dst: *mut c_void,
    dst_capacity: usize,
) -> usize {
    unsafe {
        *(*ctx).is_frame_decompression = 0;
        decode_literals_block(
            ctx,
            src.cast(),
            src_size,
            dst.cast(),
            dst_capacity,
            NOT_STREAMING,
        )
    }
}

#[inline]
fn max_short_offset() -> usize {
    if MEM_64bits() {
        usize::MAX
    } else {
        ((1usize << 26) - 1).wrapping_sub(ZSTD_REP_NUM)
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decompressBlock_internal(
    ctx: *mut ZSTD_rustBlockCtx,
    dst: *mut c_void,
    dst_capacity: usize,
    src: *const c_void,
    src_size: usize,
    streaming: c_int,
) -> usize {
    let block_max = unsafe { block_size_max(ctx) };
    if src_size > block_max {
        return ERROR(ZstdErrorCode::SrcSizeWrong);
    }
    let mut input = src.cast::<u8>();
    let lit_size =
        unsafe { decode_literals_block(ctx, input, src_size, dst.cast(), dst_capacity, streaming) };
    if ERR_isError(lit_size) || lit_size > src_size {
        return lit_size;
    }
    input = unsafe { input.add(lit_size) };
    let remaining = src_size - lit_size;
    let history_size = unsafe { total_history_size(ctx, dst.cast(), dst_capacity) };
    let potential_long_offsets = MEM_32bits() && history_size > max_short_offset();
    let mut nb_seq = 0 as c_int;
    let header_size =
        unsafe { ZSTD_rust_decodeSeqHeaders(ctx, &mut nb_seq, input.cast(), remaining) };
    if ERR_isError(header_size) || header_size > remaining {
        return header_size;
    }
    input = unsafe { input.add(header_size) };
    let sequence_size = remaining - header_size;
    if (dst.is_null() || dst_capacity == 0) && nb_seq > 0 {
        return ERROR(ZstdErrorCode::DstSizeTooSmall);
    }
    let offset_info = if potential_long_offsets
        || (unsafe { *(*ctx).ddict_is_cold } == 0
            && history_size > LONG_OFFSET_HISTORY_THRESHOLD
            && nb_seq > 8)
    {
        unsafe { get_offset_info(*(*ctx).oft_ptr, nb_seq) }
    } else {
        ZSTD_offsetInfo::default()
    };
    let mut long_offsets = potential_long_offsets;
    if long_offsets && offset_info.max_nb_additional_bits <= stream_accumulator_min() as u8 {
        long_offsets = false;
    }
    let decoder = choose_sequence_decoder(
        unsafe { (*ctx).sequence_decoder_mode },
        unsafe { *(*ctx).ddict_is_cold },
        potential_long_offsets,
        history_size,
        nb_seq,
        offset_info,
    );
    unsafe { *(*ctx).ddict_is_cold = 0 };
    unsafe {
        decompress_sequences(
            ctx,
            dst.cast(),
            dst_capacity,
            input,
            sequence_size,
            nb_seq,
            long_offsets,
            decoder == SequenceDecoder::Long,
        )
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_checkContinuity(
    ctx: *mut ZSTD_rustBlockCtx,
    dst: *const c_void,
    dst_size: usize,
) {
    unsafe {
        if dst.cast::<u8>() != *(*ctx).previous_dst_end && dst_size != 0 {
            *(*ctx).dict_end = *(*ctx).previous_dst_end;
            let prefix = *(*ctx).prefix_start;
            let previous = *(*ctx).previous_dst_end;
            let distance = address_distance(previous, prefix);
            *(*ctx).virtual_start = dst.cast::<u8>().wrapping_sub(distance);
            *(*ctx).prefix_start = dst.cast();
            *(*ctx).previous_dst_end = dst.cast();
        }
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decompressBlock_deprecated(
    ctx: *mut ZSTD_rustBlockCtx,
    dst: *mut c_void,
    dst_capacity: usize,
    src: *const c_void,
    src_size: usize,
) -> usize {
    unsafe {
        *(*ctx).is_frame_decompression = 0;
        ZSTD_rust_checkContinuity(ctx, dst.cast(), dst_capacity);
        let result = ZSTD_rust_decompressBlock_internal(
            ctx,
            dst,
            dst_capacity,
            src,
            src_size,
            NOT_STREAMING,
        );
        if !ERR_isError(result) {
            *(*ctx).previous_dst_end = ptr_add(dst.cast(), result).cast_const();
        }
        result
    }
}

/* The public and hidden block-decoder entrypoints retain their C ABI, but the
 * wrapper control flow lives here with the decoder implementation.  C only
 * projects the configuration-dependent private fields of ZSTD_DCtx. */

#[no_mangle]
pub unsafe extern "C" fn ZSTD_decodeLiteralsBlock_wrapper(
    dctx: *mut ZSTD_DCtx,
    src: *const c_void,
    src_size: usize,
    dst: *mut c_void,
    dst_capacity: usize,
) -> usize {
    let mut ctx = unsafe { block_context(dctx) };
    unsafe { ZSTD_rust_decodeLiteralsBlock_wrapper(&mut ctx, src, src_size, dst, dst_capacity) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_decodeSeqHeaders(
    dctx: *mut ZSTD_DCtx,
    nb_seq_ptr: *mut c_int,
    src: *const c_void,
    src_size: usize,
) -> usize {
    let mut ctx = unsafe { block_context(dctx) };
    unsafe { ZSTD_rust_decodeSeqHeaders(&mut ctx, nb_seq_ptr, src, src_size) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_decompressBlock_internal(
    dctx: *mut ZSTD_DCtx,
    dst: *mut c_void,
    dst_capacity: usize,
    src: *const c_void,
    src_size: usize,
    streaming: c_int,
) -> usize {
    let mut ctx = unsafe { block_context(dctx) };
    unsafe {
        ZSTD_rust_decompressBlock_internal(&mut ctx, dst, dst_capacity, src, src_size, streaming)
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_checkContinuity(
    dctx: *mut ZSTD_DCtx,
    dst: *const c_void,
    dst_size: usize,
) {
    let mut ctx = unsafe { block_context(dctx) };
    unsafe { ZSTD_rust_checkContinuity(&mut ctx, dst, dst_size) };
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_decompressBlock_deprecated(
    dctx: *mut ZSTD_DCtx,
    dst: *mut c_void,
    dst_capacity: usize,
    src: *const c_void,
    src_size: usize,
) -> usize {
    let mut ctx = unsafe { block_context(dctx) };
    unsafe { ZSTD_rust_decompressBlock_deprecated(&mut ctx, dst, dst_capacity, src, src_size) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_decompressBlock(
    dctx: *mut ZSTD_DCtx,
    dst: *mut c_void,
    dst_capacity: usize,
    src: *const c_void,
    src_size: usize,
) -> usize {
    unsafe { ZSTD_decompressBlock_deprecated(dctx, dst, dst_capacity, src, src_size) }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn offset_code_22_and_23_keep_their_22_and_23_bit_bases() {
        assert_eq!(OF_BASE[22], 0x3F_FFFD);
        assert_eq!(OF_BASE[23], 0x7F_FFFD);
        assert_eq!(OF_BASE[24], 0xFF_FFFD);

        let mut normalized = [0i16; MAX_OFF + 1];
        normalized[22] = 1 << 5;
        let mut table = [ZSTD_seqSymbol::default(); 1 + (1 << 5)];
        let mut workspace = [0u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
        unsafe {
            build_fse_table_body(
                table.as_mut_ptr(),
                normalized.as_ptr(),
                22,
                OF_BASE.as_ptr(),
                OF_BITS.as_ptr(),
                5,
                workspace.as_mut_ptr(),
                std::mem::size_of_val(&workspace),
            );
        }
        assert!(table[1..]
            .iter()
            .all(|symbol| symbol.base_value == 0x3F_FFFD));
    }

    #[test]
    fn offset_info_scales_long_share_and_handles_invalid_tables() {
        let mut table = [ZSTD_seqSymbol::default(); 1 + (1 << 5)];
        unsafe {
            set_seq_header(
                table.as_mut_ptr(),
                ZSTD_seqSymbol_header {
                    fast_mode: 0,
                    table_log: 5,
                },
            );
        }
        table[1].nb_additional_bits = 23;
        table[2].nb_additional_bits = 31;

        let info = unsafe { get_offset_info(table.as_ptr(), 1) };
        assert_eq!(info.max_nb_additional_bits, 31);
        assert_eq!(info.long_offset_share, 2 << (OFF_FSE_LOG - 5));
        assert_eq!(
            unsafe { get_offset_info(table.as_ptr(), 0) },
            ZSTD_offsetInfo::default()
        );

        unsafe {
            set_seq_header(
                table.as_mut_ptr(),
                ZSTD_seqSymbol_header {
                    fast_mode: 0,
                    table_log: OFF_FSE_LOG + 1,
                },
            );
        }
        assert_eq!(
            unsafe { get_offset_info(table.as_ptr(), 1) },
            ZSTD_offsetInfo::default()
        );
    }

    #[test]
    fn sequence_decoder_policy_preserves_forced_and_runtime_variants() {
        let minimum_share = if MEM_32bits() {
            LONG_OFFSET_MIN_SHARE_32
        } else {
            LONG_OFFSET_MIN_SHARE_64
        };
        let info = ZSTD_offsetInfo {
            long_offset_share: minimum_share,
            max_nb_additional_bits: 31,
        };

        assert_eq!(
            choose_sequence_decoder(SEQUENCE_DECODER_FORCE_SHORT, 1, true, usize::MAX, 9, info,),
            SequenceDecoder::Short
        );
        assert_eq!(
            choose_sequence_decoder(
                SEQUENCE_DECODER_FORCE_LONG,
                0,
                false,
                0,
                0,
                ZSTD_offsetInfo::default(),
            ),
            SequenceDecoder::Long
        );
        assert_eq!(
            choose_sequence_decoder(
                SEQUENCE_DECODER_RUNTIME,
                1,
                false,
                0,
                0,
                ZSTD_offsetInfo::default(),
            ),
            SequenceDecoder::Long
        );
        assert_eq!(
            choose_sequence_decoder(
                SEQUENCE_DECODER_RUNTIME,
                0,
                false,
                LONG_OFFSET_HISTORY_THRESHOLD + 1,
                9,
                info,
            ),
            SequenceDecoder::Long
        );
        assert_eq!(
            choose_sequence_decoder(
                SEQUENCE_DECODER_RUNTIME,
                0,
                false,
                LONG_OFFSET_HISTORY_THRESHOLD,
                8,
                info,
            ),
            SequenceDecoder::Short
        );
    }

    #[test]
    fn block_header_edges_keep_rle_reserved_and_truncated_contracts() {
        let mut properties = block_properties_t {
            block_type: -1,
            last_block: 99,
            orig_size: 99,
        };
        let truncated = [0u8; 2];
        let truncated_result = unsafe {
            ZSTD_getcBlockSize(
                truncated.as_ptr().cast(),
                truncated.len(),
                (&mut properties as *mut block_properties_t).cast(),
            )
        };
        assert_eq!(
            crate::errors::ERR_getErrorCode(truncated_result),
            ZstdErrorCode::SrcSizeWrong as i32
        );
        assert_eq!(properties.block_type, -1);

        let rle_header = (0x1234u32 << 3) | (1 << 1) | 1;
        let rle = [
            rle_header as u8,
            (rle_header >> 8) as u8,
            (rle_header >> 16) as u8,
        ];
        let rle_result = unsafe {
            ZSTD_getcBlockSize(
                rle.as_ptr().cast(),
                rle.len(),
                (&mut properties as *mut block_properties_t).cast(),
            )
        };
        assert_eq!(rle_result, 1);
        assert_eq!(properties.block_type, SET_RLE);
        assert_eq!(properties.last_block, 1);
        assert_eq!(properties.orig_size, 0x1234);

        let reserved_header = 3u32 << 1;
        let reserved = [
            reserved_header as u8,
            (reserved_header >> 8) as u8,
            (reserved_header >> 16) as u8,
        ];
        let reserved_result = unsafe {
            ZSTD_getcBlockSize(
                reserved.as_ptr().cast(),
                reserved.len(),
                (&mut properties as *mut block_properties_t).cast(),
            )
        };
        assert_eq!(
            crate::errors::ERR_getErrorCode(reserved_result),
            ZstdErrorCode::CorruptionDetected as i32
        );
        assert_eq!(properties.block_type, 3);
    }
}