zstd-rs/rust/src/zstd_ldm.rs

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]

//! Long distance matching.
//!
//! The C translation unit owns opaque compression-context dispatch and exports
//! the immutable gear table. This module owns gear splitting, LDM table
//! maintenance, raw-sequence generation, and raw-sequence consumption.

use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::mem::{MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readST};
use crate::xxhash::XXH64;
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::c_int;

const LDM_BATCH_SIZE: usize = 64;
const LDM_BUCKET_SIZE_LOG: u32 = 4;
const LDM_MIN_MATCH_LENGTH: u32 = 64;
const HASH_READ_SIZE: usize = 8;
const ZSTD_REP_NUM: usize = 3;
const ZSTD_WINDOW_START_INDEX: u32 = 2;

#[repr(C)]
#[derive(Clone, Copy)]
struct LdmEntry {
    offset: u32,
    checksum: u32,
}

#[repr(C)]
#[derive(Clone, Copy)]
struct RawSeq {
    offset: u32,
    lit_length: u32,
    match_length: u32,
}

#[repr(C)]
struct RawSeqStore {
    seq: *mut RawSeq,
    pos: usize,
    pos_in_sequence: usize,
    size: usize,
    capacity: usize,
}

#[repr(C)]
struct LdmParams {
    enable_ldm: c_int,
    hash_log: u32,
    bucket_size_log: u32,
    min_match_length: u32,
    hash_rate_log: u32,
    window_log: u32,
}

#[repr(C)]
struct LdmWindow {
    next_src: *const u8,
    base: *const u8,
    dict_base: *const u8,
    dict_limit: u32,
    low_limit: u32,
    nb_overflow_corrections: u32,
}

#[derive(Clone, Copy)]
struct RollingHashState {
    rolling: u64,
    stop_mask: u64,
}

#[derive(Clone, Copy)]
struct MatchCandidate {
    split: *const u8,
    hash: u32,
    checksum: u32,
    bucket: *mut LdmEntry,
}

const EMPTY_CANDIDATE: MatchCandidate = MatchCandidate {
    split: std::ptr::null(),
    hash: 0,
    checksum: 0,
    bucket: std::ptr::null_mut(),
};

unsafe extern "C" {
    fn ZSTD_ldm_rust_gearTable() -> *const u64;
    fn ZSTD_ldm_rust_prepareBlock(context: *mut c_void, anchor: *const c_void);
    fn ZSTD_ldm_rust_compressLiterals(
        context: *mut c_void,
        seq_store: *mut c_void,
        reps: *mut u32,
        src: *const c_void,
        src_size: usize,
    ) -> usize;
    fn ZSTD_ldm_rust_storeSeq(
        seq_store: *mut c_void,
        lit_length: usize,
        literals: *const c_void,
        lit_limit: *const c_void,
        off_base: u32,
        match_length: usize,
    );
    fn ZSTD_ldm_rust_setLdmSeqStore(context: *mut c_void, raw_seq_store: *const c_void);
}

#[inline]
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
    (left as usize) < (right as usize)
}

#[inline]
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
    (left as usize) > (right as usize)
}

#[inline]
unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 {
    unsafe { ptr.offset_from(base) as u32 }
}

#[inline]
fn common_bytes(word: usize) -> usize {
    let zeros = if MEM_isLittleEndian() {
        word.trailing_zeros()
    } else {
        word.leading_zeros()
    };
    (zeros / 8) as usize
}

unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
    let input_start = input;
    let word_size = size_of::<usize>();
    while unsafe { input_limit.offset_from(input) as usize } >= word_size {
        let diff =
            unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
        if diff != 0 {
            return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff);
        }
        input = input.wrapping_add(word_size);
        matched = matched.wrapping_add(word_size);
    }
    if MEM_64bits()
        && unsafe { input_limit.offset_from(input) as usize } >= 4
        && unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
    {
        input = input.wrapping_add(4);
        matched = matched.wrapping_add(4);
    }
    if unsafe { input_limit.offset_from(input) as usize } >= 2
        && unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
    {
        input = input.wrapping_add(2);
        matched = matched.wrapping_add(2);
    }
    if ptr_lt(input, input_limit) && unsafe { *input == *matched } {
        input = input.wrapping_add(1);
    }
    unsafe { input.offset_from(input_start) as usize }
}

unsafe fn count_2segments(
    input: *const u8,
    matched: *const u8,
    input_end: *const u8,
    match_end: *const u8,
    input_start: *const u8,
) -> usize {
    let match_remaining = unsafe { match_end.offset_from(matched) as usize };
    let input_remaining = unsafe { input_end.offset_from(input) as usize };
    let first_end = input.wrapping_add(match_remaining.min(input_remaining));
    let first_count = unsafe { count(input, matched, first_end) };
    if matched.wrapping_add(first_count) != match_end {
        return first_count;
    }
    first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
}

#[inline]
fn bounded(lower: u32, value: u32, upper: u32) -> u32 {
    value.max(lower).min(upper)
}

/// Return the next match-table update point using the C scalar rule.
#[no_mangle]
pub extern "C" fn ZSTD_rust_ldm_limitTableUpdate(curr: u32, next_to_update: u32) -> u32 {
    if curr > next_to_update.wrapping_add(1024) {
        curr.wrapping_sub(512u32.min(curr.wrapping_sub(next_to_update).wrapping_sub(1024)))
    } else {
        next_to_update
    }
}

#[inline]
unsafe fn ldm_bucket(hash_table: *mut LdmEntry, hash: u32, bucket_size_log: u32) -> *mut LdmEntry {
    unsafe { hash_table.add((hash as usize) << bucket_size_log) }
}

unsafe fn ldm_insert_entry(
    hash_table: *mut LdmEntry,
    bucket_offsets: *mut u8,
    hash: u32,
    entry: LdmEntry,
    bucket_size_log: u32,
) {
    let offset = unsafe { *bucket_offsets.add(hash as usize) };
    let bucket = unsafe { ldm_bucket(hash_table, hash, bucket_size_log) };
    unsafe { *bucket.add(offset as usize) = entry };
    unsafe {
        *bucket_offsets.add(hash as usize) =
            offset.wrapping_add(1) & ((1u32.wrapping_shl(bucket_size_log)).wrapping_sub(1) as u8)
    };
}

fn gear_init(params: &LdmParams) -> RollingHashState {
    let max_bits_in_mask = params.min_match_length.min(64);
    let hash_rate_log = params.hash_rate_log;
    let stop_mask = if hash_rate_log > 0 && hash_rate_log <= max_bits_in_mask {
        ((1u64 << hash_rate_log) - 1) << (max_bits_in_mask - hash_rate_log)
    } else {
        (1u64 << hash_rate_log) - 1
    };
    RollingHashState {
        /* C assigns `~(U32)0`, which is a 32-bit all-ones value. */
        rolling: u32::MAX as u64,
        stop_mask,
    }
}

/*
 * This intentionally leaves `state.rolling` unchanged: the reference C
 * routine computes a local hash but never writes it back to the state.
 */
unsafe fn gear_reset(_state: &mut RollingHashState, _data: *const u8, _min_match_length: usize) {}

unsafe fn gear_feed(
    state: &mut RollingHashState,
    gear_table: *const u64,
    data: *const u8,
    size: usize,
    splits: &mut [usize; LDM_BATCH_SIZE],
    num_splits: &mut usize,
) -> usize {
    let mut hash = state.rolling;
    let mut n = 0usize;
    while n + 3 < size {
        for _ in 0..4 {
            hash = hash
                .wrapping_shl(1)
                .wrapping_add(unsafe { *gear_table.add(*data.add(n) as usize) });
            n += 1;
            if (hash & state.stop_mask) == 0 {
                splits[*num_splits] = n;
                *num_splits += 1;
                if *num_splits == LDM_BATCH_SIZE {
                    state.rolling = hash;
                    return n;
                }
            }
        }
    }
    while n < size {
        hash = hash
            .wrapping_shl(1)
            .wrapping_add(unsafe { *gear_table.add(*data.add(n) as usize) });
        n += 1;
        if (hash & state.stop_mask) == 0 {
            splits[*num_splits] = n;
            *num_splits += 1;
            if *num_splits == LDM_BATCH_SIZE {
                state.rolling = hash;
                return n;
            }
        }
    }
    state.rolling = hash;
    n
}

unsafe fn count_backwards_match(
    mut input: *const u8,
    anchor: *const u8,
    mut matched: *const u8,
    match_base: *const u8,
) -> usize {
    let mut match_length = 0usize;
    while ptr_gt(input, anchor)
        && ptr_gt(matched, match_base)
        && unsafe { *input.wrapping_sub(1) == *matched.wrapping_sub(1) }
    {
        input = input.wrapping_sub(1);
        matched = matched.wrapping_sub(1);
        match_length += 1;
    }
    match_length
}

unsafe fn count_backwards_match_2segments(
    input: *const u8,
    anchor: *const u8,
    matched: *const u8,
    match_base: *const u8,
    ext_dict_start: *const u8,
    ext_dict_end: *const u8,
) -> usize {
    let match_length = unsafe { count_backwards_match(input, anchor, matched, match_base) };
    if matched.wrapping_sub(match_length) != match_base || match_base == ext_dict_start {
        return match_length;
    }
    match_length
        + unsafe {
            count_backwards_match(
                input.wrapping_sub(match_length),
                anchor,
                ext_dict_end,
                ext_dict_start,
            )
        }
}

fn window_has_ext_dict(window: &LdmWindow) -> bool {
    window.low_limit < window.dict_limit
}

fn window_can_overflow_correct(
    window: &LdmWindow,
    cycle_log: u32,
    max_dist: u32,
    loaded_dict_end: u32,
    src: *const u8,
) -> bool {
    let cycle_size = 1u32.wrapping_shl(cycle_log);
    let current = unsafe { index_from(window.base, src) };
    let min_index = cycle_size
        .wrapping_add(max_dist.max(cycle_size))
        .wrapping_add(ZSTD_WINDOW_START_INDEX);
    let adjustment = window.nb_overflow_corrections.wrapping_add(1);
    let adjusted = min_index.wrapping_mul(adjustment).max(min_index);
    let index_large_enough = current > adjusted;
    let dictionary_invalidated = current > max_dist.wrapping_add(loaded_dict_end);
    index_large_enough && dictionary_invalidated
}

fn window_needs_overflow_correction(
    window: &LdmWindow,
    cycle_log: u32,
    max_dist: u32,
    loaded_dict_end: u32,
    src: *const u8,
    src_end: *const u8,
    overflow_correct_frequently: bool,
) -> bool {
    if overflow_correct_frequently
        && window_can_overflow_correct(window, cycle_log, max_dist, loaded_dict_end, src)
    {
        return true;
    }
    let current = unsafe { index_from(window.base, src_end) };
    let current_max = if size_of::<usize>() == 8 {
        3500u32 * (1 << 20)
    } else {
        2000u32 * (1 << 20)
    };
    current > current_max
}

unsafe fn window_correct_overflow(
    window: &mut LdmWindow,
    cycle_log: u32,
    max_dist: u32,
    src: *const u8,
) -> u32 {
    let cycle_size = 1u32.wrapping_shl(cycle_log);
    let cycle_mask = cycle_size.wrapping_sub(1);
    let current = unsafe { index_from(window.base, src) };
    let current_cycle = current & cycle_mask;
    let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX {
        cycle_size.max(ZSTD_WINDOW_START_INDEX)
    } else {
        0
    };
    let new_current = current_cycle
        .wrapping_add(current_cycle_correction)
        .wrapping_add(max_dist.max(cycle_size));
    let correction = current.wrapping_sub(new_current);
    window.base = window.base.wrapping_add(correction as usize);
    window.dict_base = window.dict_base.wrapping_add(correction as usize);
    if window.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) {
        window.low_limit = ZSTD_WINDOW_START_INDEX;
    } else {
        window.low_limit = window.low_limit.wrapping_sub(correction);
    }
    if window.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) {
        window.dict_limit = ZSTD_WINDOW_START_INDEX;
    } else {
        window.dict_limit = window.dict_limit.wrapping_sub(correction);
    }
    window.nb_overflow_corrections = window.nb_overflow_corrections.wrapping_add(1);
    correction
}

fn window_enforce_max_dist(
    window: &mut LdmWindow,
    block_end: *const u8,
    max_dist: u32,
    loaded_dict_end: &mut u32,
) {
    let block_end_index = unsafe { index_from(window.base, block_end) };
    if block_end_index > max_dist.wrapping_add(*loaded_dict_end) {
        let new_low_limit = block_end_index.wrapping_sub(max_dist);
        if window.low_limit < new_low_limit {
            window.low_limit = new_low_limit;
        }
        if window.dict_limit < window.low_limit {
            window.dict_limit = window.low_limit;
        }
        *loaded_dict_end = 0;
    }
}

unsafe fn reduce_table(table: *mut LdmEntry, size: u32, reducer_value: u32) {
    for index in 0..size as usize {
        let entry = unsafe { table.add(index) };
        if unsafe { (*entry).offset < reducer_value } {
            unsafe { (*entry).offset = 0 };
        } else {
            unsafe { (*entry).offset = (*entry).offset.wrapping_sub(reducer_value) };
        }
    }
}

unsafe fn generate_sequences_internal(
    hash_table: *mut LdmEntry,
    bucket_offsets: *mut u8,
    window: &LdmWindow,
    raw_seq_store: *mut RawSeqStore,
    params: &LdmParams,
    gear_table: *const u64,
    src: *const u8,
    src_size: usize,
) -> usize {
    let ext_dict = window_has_ext_dict(window);
    let min_match_length = params.min_match_length as usize;
    let entries_per_bucket = 1usize << params.bucket_size_log;
    let hbits = params.hash_log.wrapping_sub(params.bucket_size_log);
    let dict_limit = window.dict_limit;
    let lowest_index = if ext_dict {
        window.low_limit
    } else {
        dict_limit
    };
    let base = window.base;
    let dict_base = window.dict_base;
    let dict_start = dict_base.wrapping_add(lowest_index as usize);
    let dict_end = dict_base.wrapping_add(dict_limit as usize);
    let low_prefix_ptr = base.wrapping_add(dict_limit as usize);
    let iend = src.wrapping_add(src_size);
    let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
    let mut anchor = src;
    let mut ip = src;

    if src_size < min_match_length {
        return src_size;
    }

    let mut hash_state = gear_init(params);
    unsafe { gear_reset(&mut hash_state, ip, min_match_length) };
    ip = ip.wrapping_add(min_match_length);

    while ptr_lt(ip, ilimit) {
        let mut splits = [0usize; LDM_BATCH_SIZE];
        let mut num_splits = 0usize;
        let hashed = unsafe {
            gear_feed(
                &mut hash_state,
                gear_table,
                ip,
                ilimit.offset_from(ip) as usize,
                &mut splits,
                &mut num_splits,
            )
        };
        let mut candidates = [EMPTY_CANDIDATE; LDM_BATCH_SIZE];
        for index in 0..num_splits {
            let split = ip
                .wrapping_add(splits[index])
                .wrapping_sub(min_match_length);
            let xxhash = unsafe { XXH64(split.cast::<c_void>(), min_match_length, 0) };
            let hash = (xxhash as u32) & ((1u32 << hbits) - 1);
            candidates[index] = MatchCandidate {
                split,
                hash,
                checksum: (xxhash >> 32) as u32,
                bucket: unsafe { ldm_bucket(hash_table, hash, params.bucket_size_log) },
            };
        }

        for candidate in candidates.iter().take(num_splits) {
            let split = candidate.split;
            let new_entry = LdmEntry {
                offset: unsafe { index_from(base, split) },
                checksum: candidate.checksum,
            };
            if ptr_lt(split, anchor) {
                unsafe {
                    ldm_insert_entry(
                        hash_table,
                        bucket_offsets,
                        candidate.hash,
                        new_entry,
                        params.bucket_size_log,
                    )
                };
                continue;
            }

            let mut forward_match_length = 0usize;
            let mut backward_match_length = 0usize;
            let mut best_match_length = 0usize;
            let mut best_offset = None;
            for entry_index in 0..entries_per_bucket {
                let entry = unsafe { *candidate.bucket.add(entry_index) };
                if entry.checksum != candidate.checksum || entry.offset <= lowest_index {
                    continue;
                }
                let (current_forward, current_backward) = if ext_dict {
                    let match_base = if entry.offset < dict_limit {
                        dict_base
                    } else {
                        base
                    };
                    let matched = match_base.wrapping_add(entry.offset as usize);
                    let match_end = if entry.offset < dict_limit {
                        dict_end
                    } else {
                        iend
                    };
                    let low_match = if entry.offset < dict_limit {
                        dict_start
                    } else {
                        low_prefix_ptr
                    };
                    let forward =
                        unsafe { count_2segments(split, matched, iend, match_end, low_prefix_ptr) };
                    if forward < min_match_length {
                        continue;
                    }
                    let backward = unsafe {
                        count_backwards_match_2segments(
                            split, anchor, matched, low_match, dict_start, dict_end,
                        )
                    };
                    (forward, backward)
                } else {
                    let matched = base.wrapping_add(entry.offset as usize);
                    let forward = unsafe { count(split, matched, iend) };
                    if forward < min_match_length {
                        continue;
                    }
                    let backward =
                        unsafe { count_backwards_match(split, anchor, matched, low_prefix_ptr) };
                    (forward, backward)
                };
                let total = current_forward + current_backward;
                if total > best_match_length {
                    best_match_length = total;
                    forward_match_length = current_forward;
                    backward_match_length = current_backward;
                    best_offset = Some(entry.offset);
                }
            }

            let Some(best_offset) = best_offset else {
                unsafe {
                    ldm_insert_entry(
                        hash_table,
                        bucket_offsets,
                        candidate.hash,
                        new_entry,
                        params.bucket_size_log,
                    )
                };
                continue;
            };

            let raw_seq_store = unsafe { &mut *raw_seq_store };
            if raw_seq_store.size == raw_seq_store.capacity {
                return ERROR(ZstdErrorCode::DstSizeTooSmall);
            }
            let sequence = unsafe { raw_seq_store.seq.add(raw_seq_store.size) };
            unsafe {
                (*sequence).lit_length = split
                    .wrapping_sub(backward_match_length)
                    .offset_from(anchor) as u32;
                (*sequence).match_length = (forward_match_length + backward_match_length) as u32;
                (*sequence).offset = index_from(base, split).wrapping_sub(best_offset);
            }
            raw_seq_store.size += 1;
            unsafe {
                ldm_insert_entry(
                    hash_table,
                    bucket_offsets,
                    candidate.hash,
                    new_entry,
                    params.bucket_size_log,
                )
            };
            anchor = split.wrapping_add(forward_match_length);
            if ptr_gt(anchor, ip.wrapping_add(hashed)) {
                unsafe {
                    gear_reset(
                        &mut hash_state,
                        anchor.wrapping_sub(min_match_length),
                        min_match_length,
                    )
                };
                ip = anchor.wrapping_sub(hashed);
                break;
            }
        }
        ip = ip.wrapping_add(hashed);
    }
    unsafe { iend.offset_from(anchor) as usize }
}

/// Rust implementation called by the C ABI wrapper for parameter adjustment.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_ldm_adjustParameters(
    params: *mut c_void,
    window_log: u32,
    strategy: c_int,
    hash_log_max: u32,
    bucket_size_log_max: u32,
    btultra: c_int,
) {
    let params = unsafe { &mut *params.cast::<LdmParams>() };
    params.window_log = window_log;
    if params.hash_rate_log == 0 {
        if params.hash_log > 0 {
            if params.window_log > params.hash_log {
                params.hash_rate_log = params.window_log - params.hash_log;
            }
        } else {
            params.hash_rate_log = 7u32.wrapping_sub((strategy / 3) as u32);
        }
    }
    if params.hash_log == 0 {
        params.hash_log = bounded(
            6,
            params.window_log.wrapping_sub(params.hash_rate_log),
            hash_log_max,
        );
    }
    if params.min_match_length == 0 {
        params.min_match_length = LDM_MIN_MATCH_LENGTH;
        if strategy >= btultra {
            params.min_match_length /= 2;
        }
    }
    if params.bucket_size_log == 0 {
        params.bucket_size_log = bounded(LDM_BUCKET_SIZE_LOG, strategy as u32, bucket_size_log_max);
    }
    params.bucket_size_log = params.bucket_size_log.min(params.hash_log);
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_ldm_getTableSize(
    params: *const c_void,
    enable_ldm: c_int,
    redzone_size: usize,
) -> usize {
    let params = unsafe { &*params.cast::<LdmParams>() };
    let hash_size = 1usize << params.hash_log;
    let bucket_log = params.bucket_size_log.min(params.hash_log);
    let bucket_size = 1usize << (params.hash_log - bucket_log);
    let alloc_size = |size: usize| {
        if size == 0 {
            0
        } else {
            size + 2 * redzone_size
        }
    };
    if enable_ldm != 0 {
        alloc_size(bucket_size) + alloc_size(hash_size * size_of::<LdmEntry>())
    } else {
        0
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_ldm_getMaxNbSeq(
    params: *const c_void,
    enable_ldm: c_int,
    max_chunk_size: usize,
) -> usize {
    let params = unsafe { &*params.cast::<LdmParams>() };
    if enable_ldm != 0 {
        max_chunk_size / params.min_match_length as usize
    } else {
        0
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_ldm_fillHashTable(
    hash_table: *mut c_void,
    bucket_offsets: *mut u8,
    base: *const u8,
    mut input: *const u8,
    input_end: *const u8,
    params: *const c_void,
) {
    let hash_table = hash_table.cast::<LdmEntry>();
    let params = unsafe { &*params.cast::<LdmParams>() };
    let min_match_length = params.min_match_length as usize;
    let hbits = params.hash_log.wrapping_sub(params.bucket_size_log);
    let input_start = input;
    let gear_table = unsafe { ZSTD_ldm_rust_gearTable() };
    let mut hash_state = gear_init(params);
    while ptr_lt(input, input_end) {
        let mut splits = [0usize; LDM_BATCH_SIZE];
        let mut num_splits = 0usize;
        let hashed = unsafe {
            gear_feed(
                &mut hash_state,
                gear_table,
                input,
                input_end.offset_from(input) as usize,
                &mut splits,
                &mut num_splits,
            )
        };
        for split_index in splits.iter().take(num_splits) {
            if input.wrapping_add(*split_index) >= input_start.wrapping_add(min_match_length) {
                let split = input
                    .wrapping_add(*split_index)
                    .wrapping_sub(min_match_length);
                let xxhash = unsafe { XXH64(split.cast::<c_void>(), min_match_length, 0) };
                let hash = (xxhash as u32) & ((1u32 << hbits) - 1);
                unsafe {
                    ldm_insert_entry(
                        hash_table,
                        bucket_offsets,
                        hash,
                        LdmEntry {
                            offset: index_from(base, split),
                            checksum: (xxhash >> 32) as u32,
                        },
                        params.bucket_size_log,
                    )
                };
            }
        }
        input = input.wrapping_add(hashed);
    }
}

/// Rust implementation called by the C ABI wrapper for LDM sequence generation.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_ldm_generateSequences(
    hash_table: *mut c_void,
    bucket_offsets: *mut u8,
    window: *mut c_void,
    loaded_dict_end: *mut u32,
    raw_seq_store: *mut c_void,
    params: *const c_void,
    src: *const c_void,
    src_size: usize,
    overflow_correct_frequently: c_int,
) -> usize {
    let hash_table = hash_table.cast::<LdmEntry>();
    let params = unsafe { &*params.cast::<LdmParams>() };
    let window = unsafe { &mut *window.cast::<LdmWindow>() };
    let raw_seq_store = raw_seq_store.cast::<RawSeqStore>();
    let max_dist = 1u32.wrapping_shl(params.window_log);
    let input = src.cast::<u8>();
    let input_end = input.wrapping_add(src_size);
    const MAX_CHUNK_SIZE: usize = 1 << 20;
    let num_chunks =
        src_size / MAX_CHUNK_SIZE + usize::from(!src_size.is_multiple_of(MAX_CHUNK_SIZE));
    let mut leftover_size = 0usize;
    let gear_table = unsafe { ZSTD_ldm_rust_gearTable() };
    for chunk in 0..num_chunks {
        let chunk_start = input.wrapping_add(chunk * MAX_CHUNK_SIZE);
        let remaining = unsafe { input_end.offset_from(chunk_start) as usize };
        let chunk_end = if remaining < MAX_CHUNK_SIZE {
            input_end
        } else {
            chunk_start.wrapping_add(MAX_CHUNK_SIZE)
        };
        let chunk_size = unsafe { chunk_end.offset_from(chunk_start) as usize };
        let raw_seq_store_ref = unsafe { &mut *raw_seq_store };
        if raw_seq_store_ref.size >= raw_seq_store_ref.capacity {
            break;
        }
        let previous_size = raw_seq_store_ref.size;
        let loaded = unsafe { &mut *loaded_dict_end };
        if window_needs_overflow_correction(
            window,
            0,
            max_dist,
            *loaded,
            chunk_start,
            chunk_end,
            overflow_correct_frequently != 0,
        ) {
            let hash_size = 1u32.wrapping_shl(params.hash_log);
            let correction = unsafe { window_correct_overflow(window, 0, max_dist, chunk_start) };
            unsafe { reduce_table(hash_table, hash_size, correction) };
            *loaded = 0;
        }
        window_enforce_max_dist(window, chunk_end, max_dist, loaded);
        let leftover = unsafe {
            generate_sequences_internal(
                hash_table,
                bucket_offsets,
                window,
                raw_seq_store,
                params,
                gear_table,
                chunk_start,
                chunk_size,
            )
        };
        if ERR_isError(leftover) {
            return leftover;
        }
        let raw_seq_store_ref = unsafe { &mut *raw_seq_store };
        if previous_size < raw_seq_store_ref.size {
            unsafe {
                (*raw_seq_store_ref.seq.add(previous_size)).lit_length =
                    (*raw_seq_store_ref.seq.add(previous_size))
                        .lit_length
                        .wrapping_add(leftover_size as u32)
            };
            leftover_size = leftover;
        } else {
            leftover_size += chunk_size;
        }
    }
    0
}

unsafe fn skip_sequences(raw_seq_store: *mut RawSeqStore, mut src_size: usize, min_match: u32) {
    let raw_seq_store = unsafe { &mut *raw_seq_store };
    while src_size > 0 && raw_seq_store.pos < raw_seq_store.size {
        let sequence = unsafe { raw_seq_store.seq.add(raw_seq_store.pos) };
        if src_size <= unsafe { (*sequence).lit_length as usize } {
            unsafe {
                (*sequence).lit_length = (*sequence).lit_length.wrapping_sub(src_size as u32)
            };
            return;
        }
        src_size -= unsafe { (*sequence).lit_length as usize };
        unsafe { (*sequence).lit_length = 0 };
        if src_size < unsafe { (*sequence).match_length as usize } {
            unsafe {
                (*sequence).match_length = (*sequence).match_length.wrapping_sub(src_size as u32)
            };
            if unsafe { (*sequence).match_length < min_match } {
                if raw_seq_store.pos + 1 < raw_seq_store.size {
                    unsafe {
                        (*sequence.add(1)).lit_length = (*sequence.add(1))
                            .lit_length
                            .wrapping_add((*sequence).match_length)
                    };
                }
                raw_seq_store.pos += 1;
            }
            return;
        }
        src_size -= unsafe { (*sequence).match_length as usize };
        unsafe { (*sequence).match_length = 0 };
        raw_seq_store.pos += 1;
    }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_ldm_skipSequences(
    raw_seq_store: *mut c_void,
    src_size: usize,
    min_match: u32,
) {
    unsafe { skip_sequences(raw_seq_store.cast::<RawSeqStore>(), src_size, min_match) };
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_ldm_skipRawSeqStoreBytes(
    raw_seq_store: *mut c_void,
    nb_bytes: usize,
) {
    let raw_seq_store = unsafe { &mut *raw_seq_store.cast::<RawSeqStore>() };
    let mut current_position = raw_seq_store.pos_in_sequence.wrapping_add(nb_bytes) as u32;
    while current_position != 0 && raw_seq_store.pos < raw_seq_store.size {
        let sequence = unsafe { *raw_seq_store.seq.add(raw_seq_store.pos) };
        if current_position >= sequence.lit_length.wrapping_add(sequence.match_length) {
            current_position = current_position
                .wrapping_sub(sequence.lit_length)
                .wrapping_sub(sequence.match_length);
            raw_seq_store.pos += 1;
        } else {
            raw_seq_store.pos_in_sequence = current_position as usize;
            break;
        }
    }
    if current_position == 0 || raw_seq_store.pos == raw_seq_store.size {
        raw_seq_store.pos_in_sequence = 0;
    }
}

unsafe fn maybe_split_sequence(
    raw_seq_store: *mut RawSeqStore,
    remaining: u32,
    min_match: u32,
) -> RawSeq {
    let raw_seq_store_ref = unsafe { &mut *raw_seq_store };
    let mut sequence = unsafe { *raw_seq_store_ref.seq.add(raw_seq_store_ref.pos) };
    if remaining >= sequence.lit_length.wrapping_add(sequence.match_length) {
        raw_seq_store_ref.pos += 1;
        return sequence;
    }
    if remaining <= sequence.lit_length {
        sequence.offset = 0;
    } else {
        sequence.match_length = remaining.wrapping_sub(sequence.lit_length);
        if sequence.match_length < min_match {
            sequence.offset = 0;
        }
    }
    unsafe { skip_sequences(raw_seq_store, remaining as usize, min_match) };
    sequence
}

/// Rust implementation called by the C ABI wrapper for LDM block integration.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_ldm_blockCompress(
    raw_seq_store: *mut c_void,
    block_context: *mut c_void,
    seq_store: *mut c_void,
    reps: *mut u32,
    src: *const c_void,
    src_size: usize,
    min_match: u32,
    use_optimal_parser: c_int,
) -> usize {
    let raw_seq_store = raw_seq_store.cast::<RawSeqStore>();
    let input = src.cast::<u8>();
    let input_end = input.wrapping_add(src_size);
    if use_optimal_parser != 0 {
        unsafe { ZSTD_ldm_rust_setLdmSeqStore(block_context, raw_seq_store.cast::<c_void>()) };
        let last_literals = unsafe {
            ZSTD_ldm_rust_compressLiterals(block_context, seq_store, reps, src, src_size)
        };
        unsafe { ZSTD_rust_ldm_skipRawSeqStoreBytes(raw_seq_store.cast::<c_void>(), src_size) };
        return last_literals;
    }

    let mut input_position = input;
    while unsafe { (*raw_seq_store).pos < (*raw_seq_store).size }
        && ptr_lt(input_position, input_end)
    {
        let sequence = unsafe {
            maybe_split_sequence(
                raw_seq_store,
                input_end.offset_from(input_position) as u32,
                min_match,
            )
        };
        if sequence.offset == 0 {
            break;
        }
        unsafe { ZSTD_ldm_rust_prepareBlock(block_context, input_position.cast::<c_void>()) };
        let new_lit_length = unsafe {
            ZSTD_ldm_rust_compressLiterals(
                block_context,
                seq_store,
                reps,
                input_position.cast::<c_void>(),
                sequence.lit_length as usize,
            )
        };
        input_position = input_position.wrapping_add(sequence.lit_length as usize);
        unsafe {
            *reps.add(2) = *reps.add(1);
            *reps.add(1) = *reps;
            *reps = sequence.offset;
            ZSTD_ldm_rust_storeSeq(
                seq_store,
                new_lit_length,
                input_position.wrapping_sub(new_lit_length).cast::<c_void>(),
                input_end.cast::<c_void>(),
                sequence.offset.wrapping_add(ZSTD_REP_NUM as u32),
                sequence.match_length as usize,
            );
        }
        input_position = input_position.wrapping_add(sequence.match_length as usize);
    }
    unsafe { ZSTD_ldm_rust_prepareBlock(block_context, input_position.cast::<c_void>()) };
    unsafe {
        ZSTD_ldm_rust_compressLiterals(
            block_context,
            seq_store,
            reps,
            input_position.cast::<c_void>(),
            input_end.offset_from(input_position) as usize,
        )
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn limit_table_update_keeps_threshold_strict() {
        assert_eq!(ZSTD_rust_ldm_limitTableUpdate(1024, 0), 0);
    }

    #[test]
    fn limit_table_update_moves_one_step_past_threshold() {
        assert_eq!(ZSTD_rust_ldm_limitTableUpdate(1025, 0), 1024);
    }

    #[test]
    fn limit_table_update_clamps_to_512() {
        assert_eq!(ZSTD_rust_ldm_limitTableUpdate(2048, 0), 1536);
    }

    #[test]
    fn limit_table_update_preserves_nonzero_next_to_update() {
        assert_eq!(ZSTD_rust_ldm_limitTableUpdate(2000, 500), 1524);
    }

    #[test]
    fn limit_table_update_wraps_u32_arithmetic() {
        assert_eq!(ZSTD_rust_ldm_limitTableUpdate(1000, u32::MAX - 511), 512);
    }

    #[test]
    fn parameter_defaults_follow_the_c_rules() {
        let mut params = LdmParams {
            enable_ldm: 1,
            hash_log: 0,
            bucket_size_log: 0,
            min_match_length: 0,
            hash_rate_log: 0,
            window_log: 0,
        };
        unsafe {
            ZSTD_rust_ldm_adjustParameters(
                (&mut params as *mut LdmParams).cast::<c_void>(),
                20,
                3,
                30,
                8,
                8,
            )
        };
        assert_eq!(params.window_log, 20);
        assert_eq!(params.hash_rate_log, 6);
        assert_eq!(params.hash_log, 14);
        assert_eq!(params.bucket_size_log, 4);
        assert_eq!(params.min_match_length, 64);
    }

    #[test]
    fn raw_sequence_skipping_merges_short_tail_matches() {
        let mut sequences = [
            RawSeq {
                offset: 8,
                lit_length: 2,
                match_length: 10,
            },
            RawSeq {
                offset: 9,
                lit_length: 1,
                match_length: 12,
            },
        ];
        let mut store = RawSeqStore {
            seq: sequences.as_mut_ptr(),
            pos: 0,
            pos_in_sequence: 0,
            size: sequences.len(),
            capacity: sequences.len(),
        };
        unsafe { skip_sequences(&mut store, 10, 4) };
        assert_eq!(store.pos, 1);
        assert_eq!(sequences[1].lit_length, 3);
    }
}