zstd-rs/rust/src/zstd_lazy.rs

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
#![allow(clippy::not_unsafe_ptr_arg_deref)]

//! Lazy, lazy2, greedy, and binary-tree block match finders.
//!
//! `ZSTD_MatchState_t` deliberately stays on the C side of the boundary.  The
//! small representation below contains only the fields that the match finders
//! use, with mutable scalar fields represented by pointers back into the C
//! match state.  This keeps the matching and sequence-generation loops in
//! Rust without making a private C layout part of the Rust ABI.

use crate::bits::ZSTD_highbit32;
use crate::mem::{
    MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST,
};
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::c_int;
use std::ptr;

const ZSTD_REP_NUM: usize = 3;
const MINMATCH: usize = 3;
const HASH_READ_SIZE: usize = 8;
const K_LAZY_SKIPPING_STEP: usize = 8;
const K_SEARCH_STRENGTH: usize = 8;
const ZSTD_DUBT_UNSORTED_MARK: u32 = 1;
const ZSTD_LAZY_DDSS_BUCKET_LOG: u32 = 2;
const ZSTD_ROW_HASH_TAG_BITS: u32 = 8;
const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1;
const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8;
const ZSTD_ROW_HASH_MAX_ENTRIES: usize = 64;
const REPCODE1_TO_OFFBASE: u32 = 1;
const OFFSET_OFFBASE: u32 = ZSTD_REP_NUM as u32;

const SEARCH_HASH_CHAIN: c_int = 0;
const SEARCH_BINARY_TREE: c_int = 1;
const SEARCH_ROW_HASH: c_int = 2;

const DICT_NO_DICT: c_int = 0;
const DICT_EXT: c_int = 1;
const DICT_MATCH_STATE: c_int = 2;
const DICT_DEDICATED: c_int = 3;

#[repr(C)]
#[derive(Clone, Copy)]
struct SeqDef {
    offBase: u32,
    litLength: u16,
    mlBase: u16,
}

/// The C shim verifies the layout of this leaf structure.  The matcher only
/// needs its sequence and literal append cursors.
#[repr(C)]
struct SeqStore_t {
    sequencesStart: *mut SeqDef,
    sequences: *mut SeqDef,
    litStart: *mut u8,
    lit: *mut u8,
    llCode: *mut u8,
    mlCode: *mut u8,
    ofCode: *mut u8,
    maxNbSeq: usize,
    maxNbLit: usize,
    longLengthType: c_int,
    longLengthPos: u32,
}

/// A field-level view of `ZSTD_MatchState_t` used by the lazy match finders.
///
/// All pointers remain owned by C.  `dictMatchState` points at a second view
/// constructed by the C shim for calls that search an attached dictionary.
#[repr(C)]
pub struct ZSTD_RustLazyState {
    hashTable: *mut u32,
    chainTable: *mut u32,
    tagTable: *mut u8,
    hashCache: *mut u32,
    base: *const u8,
    dictBase: *const u8,
    nextSrc: *const u8,
    dictLimit: u32,
    lowLimit: u32,
    loadedDictEnd: u32,
    nextToUpdate: *mut u32,
    lazySkipping: *mut c_int,
    hashLog: u32,
    chainLog: u32,
    minMatch: u32,
    searchLog: u32,
    windowLog: u32,
    rowHashLog: u32,
    hashSalt: u64,
    hashSaltEntropy: *mut u32,
    dictMatchState: *const ZSTD_RustLazyState,
}

#[inline]
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
    (left as usize) < (right as usize)
}

#[inline]
fn ptr_le(left: *const u8, right: *const u8) -> bool {
    (left as usize) <= (right as usize)
}

#[inline]
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
    (left as usize) > (right as usize)
}

#[inline]
unsafe fn index_from(base: *const u8, value: *const u8) -> u32 {
    (value as usize).wrapping_sub(base as usize) as u32
}

#[inline]
unsafe fn read32(value: *const u8) -> u32 {
    unsafe { MEM_read32(value.cast::<c_void>()) }
}

#[inline]
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
    unsafe { *table.add(index) }
}

#[inline]
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
    unsafe { *table.add(index) = value };
}

#[inline]
unsafe fn byte_get(table: *const u8, index: usize) -> u8 {
    unsafe { *table.add(index) }
}

#[inline]
unsafe fn byte_set(table: *mut u8, index: usize, value: u8) {
    unsafe { *table.add(index) = value };
}

#[inline]
unsafe fn next_to_update(state: &ZSTD_RustLazyState) -> u32 {
    unsafe { *state.nextToUpdate }
}

#[inline]
unsafe fn set_next_to_update(state: &ZSTD_RustLazyState, value: u32) {
    unsafe { *state.nextToUpdate = value };
}

#[inline]
unsafe fn lazy_skipping(state: &ZSTD_RustLazyState) -> bool {
    unsafe { *state.lazySkipping != 0 }
}

#[inline]
unsafe fn set_lazy_skipping(state: &ZSTD_RustLazyState, value: bool) {
    unsafe { *state.lazySkipping = c_int::from(value) };
}

#[inline]
unsafe fn add_hash_salt_entropy(state: &ZSTD_RustLazyState, value: u32) {
    unsafe {
        *state.hashSaltEntropy = (*state.hashSaltEntropy).wrapping_add(value);
    }
}

#[inline]
unsafe fn dict_state<'a>(state: &ZSTD_RustLazyState) -> &'a ZSTD_RustLazyState {
    debug_assert!(!state.dictMatchState.is_null());
    unsafe { &*state.dictMatchState }
}

#[inline]
fn bounded(low: u32, value: u32, high: u32) -> u32 {
    value.clamp(low, high)
}

#[inline]
fn hash_shift32(value: u32, hbits: u32) -> usize {
    if hbits == 0 {
        0
    } else {
        (value >> (32 - hbits)) as usize
    }
}

#[inline]
fn hash_shift64(value: u64, hbits: u32) -> usize {
    if hbits == 0 {
        0
    } else {
        (value >> (64 - hbits)) as usize
    }
}

#[inline]
unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize {
    match mls {
        5 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits)
        }
        6 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(16).wrapping_mul(227_718_039_650_203),
                hbits,
            )
        }
        7 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627),
                hbits,
            )
        }
        8 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits)
        }
        _ => {
            let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
            hash_shift32(value.wrapping_mul(2_654_435_761), hbits)
        }
    }
}

#[inline]
unsafe fn hash_ptr_salted(ptr: *const u8, hbits: u32, mls: u32, salt: u64) -> usize {
    match mls {
        5 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(24).wrapping_mul(889_523_592_379) ^ salt,
                hbits,
            )
        }
        6 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(16).wrapping_mul(227_718_039_650_203) ^ salt,
                hbits,
            )
        }
        7 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627) ^ salt,
                hbits,
            )
        }
        8 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463) ^ salt, hbits)
        }
        _ => {
            let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
            hash_shift32(value.wrapping_mul(2_654_435_761) ^ salt as u32, hbits)
        }
    }
}

#[inline]
fn common_bytes(word: usize) -> usize {
    let zeros = if MEM_isLittleEndian() {
        word.trailing_zeros()
    } else {
        word.leading_zeros()
    };
    (zeros / 8) as usize
}

/// Equivalent to C's `ZSTD_count()`.
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
    let input_start = input;
    let word_size = size_of::<usize>();

    while (input_limit as usize).wrapping_sub(input as usize) >= word_size {
        let diff =
            unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
        if diff != 0 {
            return (input as usize).wrapping_sub(input_start as usize) + common_bytes(diff);
        }
        input = input.wrapping_add(word_size);
        matched = matched.wrapping_add(word_size);
    }

    if MEM_64bits()
        && (input_limit as usize).wrapping_sub(input as usize) >= 4
        && unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
    {
        input = input.wrapping_add(4);
        matched = matched.wrapping_add(4);
    }
    if (input_limit as usize).wrapping_sub(input as usize) >= 2
        && unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
    {
        input = input.wrapping_add(2);
        matched = matched.wrapping_add(2);
    }
    if ptr_lt(input, input_limit) && unsafe { *matched == *input } {
        input = input.wrapping_add(1);
    }
    (input as usize).wrapping_sub(input_start as usize)
}

unsafe fn count_2segments(
    input: *const u8,
    matched: *const u8,
    input_end: *const u8,
    match_end: *const u8,
    input_start: *const u8,
) -> usize {
    let match_remaining = (match_end as usize).wrapping_sub(matched as usize);
    let input_remaining = (input_end as usize).wrapping_sub(input as usize);
    let first_end = input.wrapping_add(match_remaining.min(input_remaining));
    let first_count = unsafe { count(input, matched, first_end) };
    if matched.wrapping_add(first_count) != match_end {
        return first_count;
    }
    first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
}

#[inline]
fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
    let max_distance = 1u32.wrapping_shl(window_log);
    let within_window = if curr.wrapping_sub(dict_limit) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        dict_limit
    };
    if loaded_dict_end != 0 {
        dict_limit
    } else {
        within_window
    }
}

#[inline]
fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
    let max_distance = 1u32.wrapping_shl(window_log);
    let within_window = if curr.wrapping_sub(low_limit) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        low_limit
    };
    if loaded_dict_end != 0 {
        low_limit
    } else {
        within_window
    }
}

#[inline]
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
    prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
}

/// Stores exactly the observable bytes that C's `ZSTD_storeSeq()` writes.
unsafe fn store_seq(
    seq_store: *mut SeqStore_t,
    lit_length: usize,
    literals: *const u8,
    _lit_limit: *const u8,
    off_base: u32,
    match_length: usize,
) {
    let seq_store = unsafe { &mut *seq_store };
    let sequence = seq_store.sequences;
    debug_assert!(
        (sequence as usize).wrapping_sub(seq_store.sequencesStart as usize) / size_of::<SeqDef>()
            < seq_store.maxNbSeq
    );
    debug_assert!(lit_length <= seq_store.maxNbLit);
    debug_assert!(match_length >= MINMATCH);

    if lit_length != 0 {
        unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) };
    }
    seq_store.lit = seq_store.lit.wrapping_add(lit_length);

    let sequence_index =
        (sequence as usize).wrapping_sub(seq_store.sequencesStart as usize) / size_of::<SeqDef>();
    if lit_length > u16::MAX as usize {
        debug_assert_eq!(seq_store.longLengthType, 0);
        seq_store.longLengthType = 1;
        seq_store.longLengthPos = sequence_index as u32;
    }
    unsafe { (*sequence).litLength = lit_length as u16 };
    unsafe { (*sequence).offBase = off_base };

    let match_base = match_length - MINMATCH;
    if match_base > u16::MAX as usize {
        debug_assert_eq!(seq_store.longLengthType, 0);
        seq_store.longLengthType = 2;
        seq_store.longLengthPos = sequence_index as u32;
    }
    unsafe { (*sequence).mlBase = match_base as u16 };
    seq_store.sequences = sequence.wrapping_add(1);
}

#[inline]
fn offset_to_offbase(offset: u32) -> u32 {
    offset.wrapping_add(OFFSET_OFFBASE)
}

#[inline]
fn offbase_is_offset(offbase: usize) -> bool {
    offbase > OFFSET_OFFBASE as usize
}

#[inline]
fn offbase_to_offset(offbase: usize) -> u32 {
    (offbase as u32).wrapping_sub(OFFSET_OFFBASE)
}

#[inline]
fn match_improves(
    match_length: usize,
    best_length: usize,
    curr: u32,
    match_index: u32,
    old_offbase: usize,
    old_offbase_plus_one: bool,
) -> bool {
    let lhs = (match_length.wrapping_sub(best_length) as i32).wrapping_mul(4);
    let new_hb = ZSTD_highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32;
    let old_value = (old_offbase as u32).wrapping_add(u32::from(old_offbase_plus_one));
    let old_hb = ZSTD_highbit32(old_value) as i32;
    lhs > new_hb.wrapping_sub(old_hb)
}

/* ------------------------------------------------------------------------- */
/* Binary tree search                                                        */
/* ------------------------------------------------------------------------- */

unsafe fn update_dubt(state: &ZSTD_RustLazyState, ip: *const u8, _iend: *const u8, mls: u32) {
    let target = unsafe { index_from(state.base, ip) };
    let mut idx = unsafe { next_to_update(state) };
    let bt_log = state.chainLog - 1;
    let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1);

    while idx < target {
        let hash = unsafe { hash_ptr(state.base.wrapping_add(idx as usize), state.hashLog, mls) };
        let match_index = unsafe { table_get(state.hashTable, hash) };
        let cell = 2 * (idx & bt_mask) as usize;
        unsafe {
            table_set(state.hashTable, hash, idx);
            table_set(state.chainTable, cell, match_index);
            table_set(state.chainTable, cell + 1, ZSTD_DUBT_UNSORTED_MARK);
        }
        idx = idx.wrapping_add(1);
    }
    unsafe { set_next_to_update(state, target) };
}

unsafe fn insert_dubt1(
    state: &ZSTD_RustLazyState,
    curr: u32,
    input_end: *const u8,
    mut nb_compares: u32,
    bt_low: u32,
    dict_mode: c_int,
) {
    let bt_log = state.chainLog - 1;
    let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1);
    let mut common_smaller = 0usize;
    let mut common_larger = 0usize;
    let ip = if curr >= state.dictLimit {
        state.base.wrapping_add(curr as usize)
    } else {
        state.dictBase.wrapping_add(curr as usize)
    };
    let iend = if curr >= state.dictLimit {
        input_end
    } else {
        state.dictBase.wrapping_add(state.dictLimit as usize)
    };
    let dict_end = state.dictBase.wrapping_add(state.dictLimit as usize);
    let prefix_start = state.base.wrapping_add(state.dictLimit as usize);
    let window_valid = state.lowLimit;
    let max_distance = 1u32.wrapping_shl(state.windowLog);
    let window_low = if curr.wrapping_sub(window_valid) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        window_valid
    };

    let mut smaller_slot = 2 * (curr & bt_mask) as usize;
    let mut larger_slot = smaller_slot + 1;
    let mut smaller_real = true;
    let mut larger_real = true;
    let mut match_index = unsafe { table_get(state.chainTable, smaller_slot) };

    while nb_compares != 0 && match_index > window_low {
        let next_cell = 2 * (match_index & bt_mask) as usize;
        let mut match_length = common_smaller.min(common_larger);
        let mut matched;

        if dict_mode != DICT_EXT || match_index.wrapping_add(match_length as u32) >= state.dictLimit
        {
            matched = state.base.wrapping_add(match_index as usize);
            match_length += unsafe {
                count(
                    ip.wrapping_add(match_length),
                    matched.wrapping_add(match_length),
                    iend,
                )
            };
        } else {
            matched = state.dictBase.wrapping_add(match_index as usize);
            match_length += unsafe {
                count_2segments(
                    ip.wrapping_add(match_length),
                    matched.wrapping_add(match_length),
                    iend,
                    dict_end,
                    prefix_start,
                )
            };
            if match_index.wrapping_add(match_length as u32) >= state.dictLimit {
                matched = state.base.wrapping_add(match_index as usize);
            }
        }

        if ip.wrapping_add(match_length) == iend {
            break;
        }

        if unsafe { *matched.wrapping_add(match_length) < *ip.wrapping_add(match_length) } {
            if smaller_real {
                unsafe { table_set(state.chainTable, smaller_slot, match_index) };
            }
            common_smaller = match_length;
            if match_index <= bt_low {
                smaller_real = false;
                break;
            }
            smaller_slot = next_cell + 1;
            match_index = unsafe { table_get(state.chainTable, next_cell + 1) };
        } else {
            if larger_real {
                unsafe { table_set(state.chainTable, larger_slot, match_index) };
            }
            common_larger = match_length;
            if match_index <= bt_low {
                larger_real = false;
                break;
            }
            larger_slot = next_cell;
            match_index = unsafe { table_get(state.chainTable, next_cell) };
        }
        nb_compares = nb_compares.wrapping_sub(1);
    }

    if smaller_real {
        unsafe { table_set(state.chainTable, smaller_slot, 0) };
    }
    if larger_real {
        unsafe { table_set(state.chainTable, larger_slot, 0) };
    }
}

unsafe fn dubt_find_better_dict_match(
    state: &ZSTD_RustLazyState,
    ip: *const u8,
    iend: *const u8,
    offbase: &mut usize,
    mut best_length: usize,
    mut nb_compares: u32,
    mls: u32,
) -> usize {
    let dms = unsafe { dict_state(state) };
    let dict_hash = unsafe { hash_ptr(ip, dms.hashLog, mls) };
    let mut dict_match_index = unsafe { table_get(dms.hashTable, dict_hash) };
    let prefix_start = state.base.wrapping_add(state.dictLimit as usize);
    let curr = unsafe { index_from(state.base, ip) };
    let dict_base = dms.base;
    let dict_end = dms.nextSrc;
    let dict_high_limit = unsafe { index_from(dms.base, dms.nextSrc) };
    let dict_low_limit = dms.lowLimit;
    let dict_index_delta = state.lowLimit.wrapping_sub(dict_high_limit);
    let bt_log = dms.chainLog - 1;
    let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1);
    let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) {
        dict_low_limit
    } else {
        dict_high_limit.wrapping_sub(bt_mask)
    };
    let mut common_smaller = 0usize;
    let mut common_larger = 0usize;

    while nb_compares != 0 && dict_match_index > dict_low_limit {
        let next_cell = 2 * (dict_match_index & bt_mask) as usize;
        let mut match_length = common_smaller.min(common_larger);
        let mut matched = dict_base.wrapping_add(dict_match_index as usize);
        match_length += unsafe {
            count_2segments(
                ip.wrapping_add(match_length),
                matched.wrapping_add(match_length),
                iend,
                dict_end,
                prefix_start,
            )
        };
        if dict_match_index.wrapping_add(match_length as u32) >= dict_high_limit {
            matched = state
                .base
                .wrapping_add(dict_match_index.wrapping_add(dict_index_delta) as usize);
        }

        if match_length > best_length {
            let match_index = dict_match_index.wrapping_add(dict_index_delta);
            if match_improves(match_length, best_length, curr, match_index, *offbase, true) {
                best_length = match_length;
                *offbase = offset_to_offbase(curr.wrapping_sub(match_index)) as usize;
            }
            if ip.wrapping_add(match_length) == iend {
                break;
            }
        }

        if unsafe { *matched.wrapping_add(match_length) < *ip.wrapping_add(match_length) } {
            if dict_match_index <= bt_low {
                break;
            }
            common_smaller = match_length;
            dict_match_index = unsafe { table_get(dms.chainTable, next_cell + 1) };
        } else {
            if dict_match_index <= bt_low {
                break;
            }
            common_larger = match_length;
            dict_match_index = unsafe { table_get(dms.chainTable, next_cell) };
        }
        nb_compares = nb_compares.wrapping_sub(1);
    }
    best_length
}

unsafe fn dubt_find_best_match(
    state: &ZSTD_RustLazyState,
    ip: *const u8,
    iend: *const u8,
    offbase: &mut usize,
    mls: u32,
    dict_mode: c_int,
) -> usize {
    let hash = unsafe { hash_ptr(ip, state.hashLog, mls) };
    let mut match_index = unsafe { table_get(state.hashTable, hash) };
    let curr = unsafe { index_from(state.base, ip) };
    let window_low = lowest_match_index(state.lowLimit, state.loadedDictEnd, curr, state.windowLog);
    let bt_log = state.chainLog - 1;
    let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1);
    let bt_low = if bt_mask >= curr {
        0
    } else {
        curr.wrapping_sub(bt_mask)
    };
    let unsort_limit = bt_low.max(window_low);
    let mut next_candidate = 2 * (match_index & bt_mask) as usize;
    let mut unsorted_mark = next_candidate + 1;
    let mut nb_compares = 1u32.wrapping_shl(state.searchLog);
    let mut nb_candidates = nb_compares;
    let mut previous_candidate = 0u32;

    while match_index > unsort_limit
        && unsafe { table_get(state.chainTable, unsorted_mark) == ZSTD_DUBT_UNSORTED_MARK }
        && nb_candidates > 1
    {
        unsafe { table_set(state.chainTable, unsorted_mark, previous_candidate) };
        previous_candidate = match_index;
        match_index = unsafe { table_get(state.chainTable, next_candidate) };
        next_candidate = 2 * (match_index & bt_mask) as usize;
        unsorted_mark = next_candidate + 1;
        nb_candidates = nb_candidates.wrapping_sub(1);
    }

    if match_index > unsort_limit
        && unsafe { table_get(state.chainTable, unsorted_mark) == ZSTD_DUBT_UNSORTED_MARK }
    {
        unsafe {
            table_set(state.chainTable, next_candidate, 0);
            table_set(state.chainTable, unsorted_mark, 0);
        }
    }

    match_index = previous_candidate;
    while match_index != 0 {
        let next_candidate_idx =
            unsafe { table_get(state.chainTable, 2 * (match_index & bt_mask) as usize + 1) };
        unsafe {
            insert_dubt1(
                state,
                match_index,
                iend,
                nb_candidates,
                unsort_limit,
                dict_mode,
            )
        };
        match_index = next_candidate_idx;
        nb_candidates = nb_candidates.wrapping_add(1);
    }

    let mut common_smaller = 0usize;
    let mut common_larger = 0usize;
    let dict_base = state.dictBase;
    let dict_limit = state.dictLimit;
    let dict_end = dict_base.wrapping_add(dict_limit as usize);
    let prefix_start = state.base.wrapping_add(dict_limit as usize);
    let mut smaller_slot = 2 * (curr & bt_mask) as usize;
    let mut larger_slot = smaller_slot + 1;
    let mut smaller_real = true;
    let mut larger_real = true;
    let mut match_end_idx = curr.wrapping_add(HASH_READ_SIZE as u32 + 1);
    let mut best_length = 0usize;

    match_index = unsafe { table_get(state.hashTable, hash) };
    unsafe { table_set(state.hashTable, hash, curr) };

    while nb_compares != 0 && match_index > window_low {
        let next_cell = 2 * (match_index & bt_mask) as usize;
        let mut match_length = common_smaller.min(common_larger);
        let mut matched;

        if dict_mode != DICT_EXT || match_index.wrapping_add(match_length as u32) >= dict_limit {
            matched = state.base.wrapping_add(match_index as usize);
            match_length += unsafe {
                count(
                    ip.wrapping_add(match_length),
                    matched.wrapping_add(match_length),
                    iend,
                )
            };
        } else {
            matched = dict_base.wrapping_add(match_index as usize);
            match_length += unsafe {
                count_2segments(
                    ip.wrapping_add(match_length),
                    matched.wrapping_add(match_length),
                    iend,
                    dict_end,
                    prefix_start,
                )
            };
            if match_index.wrapping_add(match_length as u32) >= dict_limit {
                matched = state.base.wrapping_add(match_index as usize);
            }
        }

        if match_length > best_length {
            if match_length > match_end_idx.wrapping_sub(match_index) as usize {
                match_end_idx = match_index.wrapping_add(match_length as u32);
            }
            if match_improves(
                match_length,
                best_length,
                curr,
                match_index,
                *offbase,
                false,
            ) {
                best_length = match_length;
                *offbase = offset_to_offbase(curr.wrapping_sub(match_index)) as usize;
            }
            if ip.wrapping_add(match_length) == iend {
                if dict_mode == DICT_MATCH_STATE {
                    nb_compares = 0;
                }
                break;
            }
        }

        if unsafe { *matched.wrapping_add(match_length) < *ip.wrapping_add(match_length) } {
            if smaller_real {
                unsafe { table_set(state.chainTable, smaller_slot, match_index) };
            }
            common_smaller = match_length;
            if match_index <= bt_low {
                smaller_real = false;
                break;
            }
            smaller_slot = next_cell + 1;
            match_index = unsafe { table_get(state.chainTable, next_cell + 1) };
        } else {
            if larger_real {
                unsafe { table_set(state.chainTable, larger_slot, match_index) };
            }
            common_larger = match_length;
            if match_index <= bt_low {
                larger_real = false;
                break;
            }
            larger_slot = next_cell;
            match_index = unsafe { table_get(state.chainTable, next_cell) };
        }
        nb_compares = nb_compares.wrapping_sub(1);
    }

    if smaller_real {
        unsafe { table_set(state.chainTable, smaller_slot, 0) };
    }
    if larger_real {
        unsafe { table_set(state.chainTable, larger_slot, 0) };
    }

    if dict_mode == DICT_MATCH_STATE && nb_compares != 0 {
        best_length = unsafe {
            dubt_find_better_dict_match(state, ip, iend, offbase, best_length, nb_compares, mls)
        };
    }

    unsafe { set_next_to_update(state, match_end_idx.wrapping_sub(HASH_READ_SIZE as u32)) };
    best_length
}

unsafe fn bt_find_best_match(
    state: &ZSTD_RustLazyState,
    ip: *const u8,
    iend: *const u8,
    offbase: &mut usize,
    mls: u32,
    dict_mode: c_int,
) -> usize {
    if ptr_lt(
        ip,
        state
            .base
            .wrapping_add(unsafe { next_to_update(state) } as usize),
    ) {
        return 0;
    }
    unsafe { update_dubt(state, ip, iend, mls) };
    unsafe { dubt_find_best_match(state, ip, iend, offbase, mls, dict_mode) }
}

/* ------------------------------------------------------------------------- */
/* Dedicated dictionary search                                                */
/* ------------------------------------------------------------------------- */

unsafe fn dedicated_dict_search_load_dictionary(state: &ZSTD_RustLazyState, ip: *const u8) {
    let target = unsafe { index_from(state.base, ip) };
    let chain_size = 1u32.wrapping_shl(state.chainLog);
    let mut idx = unsafe { next_to_update(state) };
    let min_chain = if chain_size < target.wrapping_sub(idx) {
        target.wrapping_sub(chain_size)
    } else {
        idx
    };
    let bucket_size = 1u32 << ZSTD_LAZY_DDSS_BUCKET_LOG;
    let cache_size = bucket_size - 1;
    let chain_attempts = (1u32 << state.searchLog).wrapping_sub(cache_size);
    let chain_limit = chain_attempts.min(255);
    let hash_log = state.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
    let tmp_hash_table = state.hashTable;
    let tmp_chain_table = state
        .hashTable
        .wrapping_add((1usize).wrapping_shl(hash_log));
    let tmp_chain_size = ((1u32 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1).wrapping_shl(hash_log);
    let tmp_min_chain = if tmp_chain_size < target {
        target.wrapping_sub(tmp_chain_size)
    } else {
        idx
    };

    while idx < target {
        let hash = unsafe {
            hash_ptr(
                state.base.wrapping_add(idx as usize),
                hash_log,
                state.minMatch,
            )
        };
        if idx >= tmp_min_chain {
            let previous = unsafe { table_get(state.hashTable, hash) };
            unsafe {
                table_set(
                    tmp_chain_table,
                    idx.wrapping_sub(tmp_min_chain) as usize,
                    previous,
                )
            };
        }
        unsafe { table_set(tmp_hash_table, hash, idx) };
        idx = idx.wrapping_add(1);
    }

    let mut chain_pos = 0u32;
    let mut hash_idx = 0u32;
    while hash_idx < (1u32 << hash_log) {
        let mut count = 0u32;
        let mut count_beyond_min_chain = 0u32;
        let mut chain_index = unsafe { table_get(tmp_hash_table, hash_idx as usize) };
        while chain_index >= tmp_min_chain && count < cache_size {
            if chain_index < min_chain {
                count_beyond_min_chain = count_beyond_min_chain.wrapping_add(1);
            }
            chain_index = unsafe {
                table_get(
                    tmp_chain_table,
                    chain_index.wrapping_sub(tmp_min_chain) as usize,
                )
            };
            count = count.wrapping_add(1);
        }
        if count == cache_size {
            count = 0;
            while count < chain_limit {
                if chain_index < min_chain {
                    if chain_index == 0 || count_beyond_min_chain.wrapping_add(1) > cache_size {
                        break;
                    }
                    count_beyond_min_chain = count_beyond_min_chain.wrapping_add(1);
                }
                unsafe { table_set(state.chainTable, chain_pos as usize, chain_index) };
                chain_pos = chain_pos.wrapping_add(1);
                count = count.wrapping_add(1);
                if chain_index < tmp_min_chain {
                    break;
                }
                chain_index = unsafe {
                    table_get(
                        tmp_chain_table,
                        chain_index.wrapping_sub(tmp_min_chain) as usize,
                    )
                };
            }
        } else {
            count = 0;
        }
        if count != 0 {
            unsafe {
                table_set(
                    tmp_hash_table,
                    hash_idx as usize,
                    ((chain_pos.wrapping_sub(count)) << 8).wrapping_add(count),
                )
            };
        } else {
            unsafe { table_set(tmp_hash_table, hash_idx as usize, 0) };
        }
        hash_idx = hash_idx.wrapping_add(1);
    }

    hash_idx = 1u32 << hash_log;
    while hash_idx != 0 {
        hash_idx = hash_idx.wrapping_sub(1);
        let bucket_idx = hash_idx << ZSTD_LAZY_DDSS_BUCKET_LOG;
        let packed = unsafe { table_get(tmp_hash_table, hash_idx as usize) };
        let mut i = 0u32;
        while i < cache_size {
            unsafe { table_set(state.hashTable, bucket_idx.wrapping_add(i) as usize, 0) };
            i = i.wrapping_add(1);
        }
        unsafe {
            table_set(
                state.hashTable,
                bucket_idx.wrapping_add(bucket_size).wrapping_sub(1) as usize,
                packed,
            )
        };
    }

    idx = unsafe { next_to_update(state) };
    while idx < target {
        let hash = unsafe {
            hash_ptr(
                state.base.wrapping_add(idx as usize),
                hash_log,
                state.minMatch,
            )
        } << ZSTD_LAZY_DDSS_BUCKET_LOG;
        let mut i = cache_size.wrapping_sub(1);
        while i != 0 {
            let previous =
                unsafe { table_get(state.hashTable, hash.wrapping_add((i - 1) as usize)) };
            unsafe { table_set(state.hashTable, hash.wrapping_add(i as usize), previous) };
            i = i.wrapping_sub(1);
        }
        unsafe { table_set(state.hashTable, hash, idx) };
        idx = idx.wrapping_add(1);
    }
    unsafe { set_next_to_update(state, target) };
}

unsafe fn dedicated_dict_search(
    offset: &mut usize,
    mut ml: usize,
    nb_attempts: u32,
    dms: &ZSTD_RustLazyState,
    ip: *const u8,
    ilimit: *const u8,
    prefix_start: *const u8,
    curr: u32,
    dict_limit: u32,
    dds_idx: usize,
) -> usize {
    let dds_lowest_index = dms.dictLimit;
    let dds_base = dms.base;
    let dds_end = dms.nextSrc;
    let dds_size = unsafe { index_from(dds_base, dds_end) };
    let dds_index_delta = dict_limit.wrapping_sub(dds_size);
    let bucket_size = 1u32 << ZSTD_LAZY_DDSS_BUCKET_LOG;
    let bucket_limit = nb_attempts.min(bucket_size - 1);
    let mut dds_attempt = 0u32;

    while dds_attempt < bucket_limit {
        let match_index = unsafe { table_get(dms.hashTable, dds_idx + dds_attempt as usize) };
        if match_index == 0 {
            return ml;
        }
        debug_assert!(match_index >= dds_lowest_index);
        let matched = dds_base.wrapping_add(match_index as usize);
        let mut current_ml = 0usize;
        if unsafe { read32(matched) == read32(ip) } {
            current_ml = unsafe {
                count_2segments(
                    ip.wrapping_add(4),
                    matched.wrapping_add(4),
                    ilimit,
                    dds_end,
                    prefix_start,
                ) + 4
            };
        }
        if current_ml > ml {
            ml = current_ml;
            *offset =
                offset_to_offbase(curr.wrapping_sub(match_index.wrapping_add(dds_index_delta)))
                    as usize;
            if ip.wrapping_add(current_ml) == ilimit {
                return ml;
            }
        }
        dds_attempt = dds_attempt.wrapping_add(1);
    }

    let packed = unsafe { table_get(dms.hashTable, dds_idx + (bucket_size - 1) as usize) };
    let mut chain_index = packed >> 8;
    let chain_length = packed & 0xff;
    let chain_attempts = nb_attempts.wrapping_sub(dds_attempt);
    let chain_limit = chain_attempts.min(chain_length);
    let mut chain_attempt = 0u32;
    while chain_attempt < chain_limit {
        let match_index = unsafe { table_get(dms.chainTable, chain_index as usize) };
        let matched = dds_base.wrapping_add(match_index as usize);
        let mut current_ml = 0usize;
        if unsafe { read32(matched) == read32(ip) } {
            current_ml = unsafe {
                count_2segments(
                    ip.wrapping_add(4),
                    matched.wrapping_add(4),
                    ilimit,
                    dds_end,
                    prefix_start,
                ) + 4
            };
        }
        if current_ml > ml {
            ml = current_ml;
            *offset =
                offset_to_offbase(curr.wrapping_sub(match_index.wrapping_add(dds_index_delta)))
                    as usize;
            if ip.wrapping_add(current_ml) == ilimit {
                break;
            }
        }
        chain_index = chain_index.wrapping_add(1);
        chain_attempt = chain_attempt.wrapping_add(1);
    }
    ml
}

/* ------------------------------------------------------------------------- */
/* Hash-chain search                                                         */
/* ------------------------------------------------------------------------- */

unsafe fn insert_and_find_first_index_internal(
    state: &ZSTD_RustLazyState,
    ip: *const u8,
    mls: u32,
    lazy_skip: bool,
) -> u32 {
    let chain_mask = 1u32.wrapping_shl(state.chainLog).wrapping_sub(1);
    let target = unsafe { index_from(state.base, ip) };
    let mut idx = unsafe { next_to_update(state) };

    while idx < target {
        let hash = unsafe { hash_ptr(state.base.wrapping_add(idx as usize), state.hashLog, mls) };
        let previous = unsafe { table_get(state.hashTable, hash) };
        unsafe {
            table_set(state.chainTable, (idx & chain_mask) as usize, previous);
            table_set(state.hashTable, hash, idx);
        }
        idx = idx.wrapping_add(1);
        if lazy_skip {
            break;
        }
    }
    unsafe { set_next_to_update(state, target) };
    let hash = unsafe { hash_ptr(ip, state.hashLog, mls) };
    unsafe { table_get(state.hashTable, hash) }
}

unsafe fn hc_find_best_match(
    state: &ZSTD_RustLazyState,
    ip: *const u8,
    ilimit: *const u8,
    offset: &mut usize,
    mls: u32,
    dict_mode: c_int,
) -> usize {
    let chain_size = 1u32.wrapping_shl(state.chainLog);
    let chain_mask = chain_size.wrapping_sub(1);
    let base = state.base;
    let dict_base = state.dictBase;
    let dict_limit = state.dictLimit;
    let prefix_start = base.wrapping_add(dict_limit as usize);
    let dict_end = dict_base.wrapping_add(dict_limit as usize);
    let curr = unsafe { index_from(base, ip) };
    let max_distance = 1u32.wrapping_shl(state.windowLog);
    let lowest_valid = state.lowLimit;
    let within_max_distance = if curr.wrapping_sub(lowest_valid) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        lowest_valid
    };
    let low_limit = if state.loadedDictEnd != 0 {
        lowest_valid
    } else {
        within_max_distance
    };
    let min_chain = if curr > chain_size {
        curr.wrapping_sub(chain_size)
    } else {
        0
    };
    let mut nb_attempts = 1u32.wrapping_shl(state.searchLog);
    let mut ml = 3usize;

    let dds_hash_log = if dict_mode == DICT_DEDICATED {
        unsafe { dict_state(state) }.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG
    } else {
        0
    };
    let dds_idx = if dict_mode == DICT_DEDICATED {
        (unsafe { hash_ptr(ip, dds_hash_log, mls) }) << ZSTD_LAZY_DDSS_BUCKET_LOG
    } else {
        0
    };

    let mut match_index =
        unsafe { insert_and_find_first_index_internal(state, ip, mls, lazy_skipping(state)) };
    while match_index >= low_limit && nb_attempts != 0 {
        let mut current_ml = 0usize;
        if dict_mode != DICT_EXT || match_index >= dict_limit {
            let matched = base.wrapping_add(match_index as usize);
            if unsafe { read32(matched.wrapping_add(ml - 3)) == read32(ip.wrapping_add(ml - 3)) } {
                current_ml = unsafe { count(ip, matched, ilimit) };
            }
        } else {
            let matched = dict_base.wrapping_add(match_index as usize);
            if unsafe { read32(matched) == read32(ip) } {
                current_ml = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        matched.wrapping_add(4),
                        ilimit,
                        dict_end,
                        prefix_start,
                    ) + 4
                };
            }
        }

        if current_ml > ml {
            ml = current_ml;
            *offset = offset_to_offbase(curr.wrapping_sub(match_index)) as usize;
            if ip.wrapping_add(current_ml) == ilimit {
                break;
            }
        }
        if match_index <= min_chain {
            break;
        }
        match_index = unsafe { table_get(state.chainTable, (match_index & chain_mask) as usize) };
        nb_attempts = nb_attempts.wrapping_sub(1);
    }

    if dict_mode == DICT_DEDICATED {
        ml = unsafe {
            dedicated_dict_search(
                offset,
                ml,
                nb_attempts,
                dict_state(state),
                ip,
                ilimit,
                prefix_start,
                curr,
                dict_limit,
                dds_idx,
            )
        };
    } else if dict_mode == DICT_MATCH_STATE {
        let dms = unsafe { dict_state(state) };
        let dms_chain_size = 1u32.wrapping_shl(dms.chainLog);
        let dms_chain_mask = dms_chain_size.wrapping_sub(1);
        let dms_lowest_index = dms.dictLimit;
        let dms_base = dms.base;
        let dms_end = dms.nextSrc;
        let dms_size = unsafe { index_from(dms_base, dms_end) };
        let dms_index_delta = dict_limit.wrapping_sub(dms_size);
        let dms_min_chain = if dms_size > dms_chain_size {
            dms_size.wrapping_sub(dms_chain_size)
        } else {
            0
        };

        match_index = unsafe { table_get(dms.hashTable, hash_ptr(ip, dms.hashLog, mls)) };
        while match_index >= dms_lowest_index && nb_attempts != 0 {
            let matched = dms_base.wrapping_add(match_index as usize);
            let mut current_ml = 0usize;
            if unsafe { read32(matched) == read32(ip) } {
                current_ml = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        matched.wrapping_add(4),
                        ilimit,
                        dms_end,
                        prefix_start,
                    ) + 4
                };
            }
            if current_ml > ml {
                ml = current_ml;
                *offset =
                    offset_to_offbase(curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)))
                        as usize;
                if ip.wrapping_add(current_ml) == ilimit {
                    break;
                }
            }
            if match_index <= dms_min_chain {
                break;
            }
            match_index =
                unsafe { table_get(dms.chainTable, (match_index & dms_chain_mask) as usize) };
            nb_attempts = nb_attempts.wrapping_sub(1);
        }
    }
    ml
}

/* ------------------------------------------------------------------------- */
/* Row hash search                                                           */
/* ------------------------------------------------------------------------- */

#[inline]
unsafe fn row_next_index(tag_row: *mut u8, row_mask: u32) -> u32 {
    let mut next = unsafe { ((*tag_row).wrapping_sub(1) as u32) & row_mask };
    if next == 0 {
        next = next.wrapping_add(row_mask);
    }
    unsafe { *tag_row = next as u8 };
    next
}

unsafe fn row_fill_hash_cache(
    state: &ZSTD_RustLazyState,
    base: *const u8,
    _row_log: u32,
    mls: u32,
    mut idx: u32,
    ilimit: *const u8,
) {
    let hash_log = state.rowHashLog;
    let at = base.wrapping_add(idx as usize);
    let max_to_fill = if ptr_gt(at, ilimit) {
        0
    } else {
        ((ilimit as usize).wrapping_sub(at as usize)).wrapping_add(1) as u32
    };
    let limit = idx.wrapping_add((ZSTD_ROW_HASH_CACHE_SIZE as u32).min(max_to_fill));
    while idx < limit {
        let hash = unsafe {
            hash_ptr_salted(
                base.wrapping_add(idx as usize),
                hash_log + ZSTD_ROW_HASH_TAG_BITS,
                mls,
                state.hashSalt,
            ) as u32
        };
        unsafe {
            table_set(
                state.hashCache,
                (idx as usize) & (ZSTD_ROW_HASH_CACHE_SIZE - 1),
                hash,
            )
        };
        idx = idx.wrapping_add(1);
    }
}

unsafe fn row_next_cached_hash(
    cache: *mut u32,
    _hash_table: *const u32,
    _tag_table: *const u8,
    base: *const u8,
    idx: u32,
    hash_log: u32,
    _row_log: u32,
    mls: u32,
    hash_salt: u64,
) -> u32 {
    let new_hash = unsafe {
        hash_ptr_salted(
            base.wrapping_add(idx as usize + ZSTD_ROW_HASH_CACHE_SIZE),
            hash_log + ZSTD_ROW_HASH_TAG_BITS,
            mls,
            hash_salt,
        ) as u32
    };
    let slot = (idx as usize) & (ZSTD_ROW_HASH_CACHE_SIZE - 1);
    let hash = unsafe { table_get(cache, slot) };
    unsafe { table_set(cache, slot, new_hash) };
    hash
}

unsafe fn row_update_internal_impl(
    state: &ZSTD_RustLazyState,
    mut update_start: u32,
    update_end: u32,
    mls: u32,
    row_log: u32,
    row_mask: u32,
    use_cache: bool,
) {
    while update_start < update_end {
        let hash = if use_cache {
            unsafe {
                row_next_cached_hash(
                    state.hashCache,
                    state.hashTable,
                    state.tagTable,
                    state.base,
                    update_start,
                    state.rowHashLog,
                    row_log,
                    mls,
                    state.hashSalt,
                )
            }
        } else {
            unsafe {
                hash_ptr_salted(
                    state.base.wrapping_add(update_start as usize),
                    state.rowHashLog + ZSTD_ROW_HASH_TAG_BITS,
                    mls,
                    state.hashSalt,
                ) as u32
            }
        };
        let rel_row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log;
        let row = state.hashTable.wrapping_add(rel_row as usize);
        let tag_row = state.tagTable.wrapping_add(rel_row as usize);
        let pos = unsafe { row_next_index(tag_row, row_mask) };
        unsafe {
            byte_set(tag_row, pos as usize, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8);
            table_set(row, pos as usize, update_start);
        }
        update_start = update_start.wrapping_add(1);
    }
}

unsafe fn row_update_internal(
    state: &ZSTD_RustLazyState,
    ip: *const u8,
    mls: u32,
    row_log: u32,
    row_mask: u32,
    use_cache: bool,
) {
    let mut idx = unsafe { next_to_update(state) };
    let target = unsafe { index_from(state.base, ip) };
    const SKIP_THRESHOLD: u32 = 384;
    const MAX_START: u32 = 96;
    const MAX_END: u32 = 32;

    if use_cache && target.wrapping_sub(idx) > SKIP_THRESHOLD {
        let bound = idx.wrapping_add(MAX_START);
        unsafe { row_update_internal_impl(state, idx, bound, mls, row_log, row_mask, use_cache) };
        idx = target.wrapping_sub(MAX_END);
        unsafe { row_fill_hash_cache(state, state.base, row_log, mls, idx, ip.wrapping_add(1)) };
    }
    unsafe { row_update_internal_impl(state, idx, target, mls, row_log, row_mask, use_cache) };
    unsafe { set_next_to_update(state, target) };
}

/// Returns matching tag positions in the same circular-buffer order as the
/// C SSE/SWAR mask iterator.  The scalar version avoids architecture-specific
/// vector intrinsics while preserving the selected candidate order.
unsafe fn row_matching_positions(
    tag_row: *const u8,
    tag: u8,
    head: u32,
    row_mask: u32,
    row_entries: u32,
    out: &mut [u32; ZSTD_ROW_HASH_MAX_ENTRIES],
) -> usize {
    let mut len = 0usize;
    let mut relative = 0u32;
    while relative < row_entries {
        let pos = head.wrapping_add(relative) & row_mask;
        if unsafe { byte_get(tag_row, pos as usize) } == tag {
            out[len] = pos;
            len += 1;
        }
        relative = relative.wrapping_add(1);
    }
    len
}

unsafe fn row_find_best_match(
    state: &ZSTD_RustLazyState,
    ip: *const u8,
    ilimit: *const u8,
    offset: &mut usize,
    mls: u32,
    dict_mode: c_int,
    row_log: u32,
) -> usize {
    let base = state.base;
    let dict_base = state.dictBase;
    let dict_limit = state.dictLimit;
    let prefix_start = base.wrapping_add(dict_limit as usize);
    let dict_end = dict_base.wrapping_add(dict_limit as usize);
    let curr = unsafe { index_from(base, ip) };
    let max_distance = 1u32.wrapping_shl(state.windowLog);
    let lowest_valid = state.lowLimit;
    let within_max_distance = if curr.wrapping_sub(lowest_valid) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        lowest_valid
    };
    let low_limit = if state.loadedDictEnd != 0 {
        lowest_valid
    } else {
        within_max_distance
    };
    let row_entries = 1u32 << row_log;
    let row_mask = row_entries - 1;
    let capped_search_log = state.searchLog.min(row_log);
    let mut nb_attempts = 1u32 << capped_search_log;
    let mut ml = 3usize;

    let (mut dds_idx, mut dds_extra_attempts) = (0usize, 0u32);
    if dict_mode == DICT_DEDICATED {
        let dms = unsafe { dict_state(state) };
        let dds_hash_log = dms.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
        dds_idx = unsafe { hash_ptr(ip, dds_hash_log, mls) } << ZSTD_LAZY_DDSS_BUCKET_LOG;
        dds_extra_attempts = if state.searchLog > row_log {
            1u32 << (state.searchLog - row_log)
        } else {
            0
        };
    }

    let mut dms_tag = 0u32;
    let mut dms_row = ptr::null_mut::<u32>();
    let mut dms_tag_row = ptr::null_mut::<u8>();
    if dict_mode == DICT_MATCH_STATE {
        let dms = unsafe { dict_state(state) };
        let dms_hash = unsafe { hash_ptr(ip, dms.rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls) as u32 };
        let dms_rel_row = (dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log;
        dms_tag = dms_hash & ZSTD_ROW_HASH_TAG_MASK;
        dms_tag_row = dms.tagTable.wrapping_add(dms_rel_row as usize);
        dms_row = dms.hashTable.wrapping_add(dms_rel_row as usize);
    }

    let hash = if !unsafe { lazy_skipping(state) } {
        unsafe { row_update_internal(state, ip, mls, row_log, row_mask, true) };
        unsafe {
            row_next_cached_hash(
                state.hashCache,
                state.hashTable,
                state.tagTable,
                base,
                curr,
                state.rowHashLog,
                row_log,
                mls,
                state.hashSalt,
            )
        }
    } else {
        let h = unsafe {
            hash_ptr_salted(
                ip,
                state.rowHashLog + ZSTD_ROW_HASH_TAG_BITS,
                mls,
                state.hashSalt,
            ) as u32
        };
        unsafe { set_next_to_update(state, curr) };
        h
    };
    unsafe { add_hash_salt_entropy(state, hash) };

    let rel_row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log;
    let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8;
    let row = state.hashTable.wrapping_add(rel_row as usize);
    let tag_row = state.tagTable.wrapping_add(rel_row as usize);
    let head = unsafe { byte_get(tag_row, 0) as u32 } & row_mask;
    let mut positions = [0u32; ZSTD_ROW_HASH_MAX_ENTRIES];
    let position_len = unsafe {
        row_matching_positions(tag_row, tag, head, row_mask, row_entries, &mut positions)
    };
    let mut match_buffer = [0u32; ZSTD_ROW_HASH_MAX_ENTRIES];
    let mut num_matches = 0usize;
    for pos in positions[..position_len].iter().copied() {
        if nb_attempts == 0 {
            break;
        }
        if pos == 0 {
            continue;
        }
        let match_index = unsafe { table_get(row, pos as usize) };
        if match_index < low_limit {
            break;
        }
        match_buffer[num_matches] = match_index;
        num_matches += 1;
        nb_attempts = nb_attempts.wrapping_sub(1);
    }

    let insert_pos = unsafe { row_next_index(tag_row, row_mask) };
    unsafe {
        byte_set(tag_row, insert_pos as usize, tag);
        let index = next_to_update(state);
        table_set(row, insert_pos as usize, index);
        set_next_to_update(state, index.wrapping_add(1));
    }

    for match_index in match_buffer[..num_matches].iter().copied() {
        let mut current_ml = 0usize;
        if dict_mode != DICT_EXT || match_index >= dict_limit {
            let matched = base.wrapping_add(match_index as usize);
            if unsafe { read32(matched.wrapping_add(ml - 3)) == read32(ip.wrapping_add(ml - 3)) } {
                current_ml = unsafe { count(ip, matched, ilimit) };
            }
        } else {
            let matched = dict_base.wrapping_add(match_index as usize);
            if unsafe { read32(matched) == read32(ip) } {
                current_ml = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        matched.wrapping_add(4),
                        ilimit,
                        dict_end,
                        prefix_start,
                    ) + 4
                };
            }
        }
        if current_ml > ml {
            ml = current_ml;
            *offset = offset_to_offbase(curr.wrapping_sub(match_index)) as usize;
            if ip.wrapping_add(current_ml) == ilimit {
                break;
            }
        }
    }

    if dict_mode == DICT_DEDICATED {
        ml = unsafe {
            dedicated_dict_search(
                offset,
                ml,
                nb_attempts.wrapping_add(dds_extra_attempts),
                dict_state(state),
                ip,
                ilimit,
                prefix_start,
                curr,
                dict_limit,
                dds_idx,
            )
        };
    } else if dict_mode == DICT_MATCH_STATE {
        let dms = unsafe { dict_state(state) };
        let dms_lowest_index = dms.dictLimit;
        let dms_base = dms.base;
        let dms_end = dms.nextSrc;
        let dms_size = unsafe { index_from(dms_base, dms_end) };
        let dms_index_delta = dict_limit.wrapping_sub(dms_size);
        let dms_head = unsafe { byte_get(dms_tag_row, 0) as u32 } & row_mask;
        let mut dms_positions = [0u32; ZSTD_ROW_HASH_MAX_ENTRIES];
        let dms_position_len = unsafe {
            row_matching_positions(
                dms_tag_row,
                dms_tag as u8,
                dms_head,
                row_mask,
                row_entries,
                &mut dms_positions,
            )
        };
        let mut dms_matches = [0u32; ZSTD_ROW_HASH_MAX_ENTRIES];
        let mut dms_count = 0usize;
        for pos in dms_positions[..dms_position_len].iter().copied() {
            if nb_attempts == 0 {
                break;
            }
            if pos == 0 {
                continue;
            }
            let match_index = unsafe { table_get(dms_row, pos as usize) };
            if match_index < dms_lowest_index {
                break;
            }
            dms_matches[dms_count] = match_index;
            dms_count += 1;
            nb_attempts = nb_attempts.wrapping_sub(1);
        }
        for match_index in dms_matches[..dms_count].iter().copied() {
            let matched = dms_base.wrapping_add(match_index as usize);
            let mut current_ml = 0usize;
            if unsafe { read32(matched) == read32(ip) } {
                current_ml = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        matched.wrapping_add(4),
                        ilimit,
                        dms_end,
                        prefix_start,
                    ) + 4
                };
            }
            if current_ml > ml {
                ml = current_ml;
                *offset =
                    offset_to_offbase(curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)))
                        as usize;
                if ip.wrapping_add(current_ml) == ilimit {
                    break;
                }
            }
        }
    }
    ml
}

unsafe fn search_max(
    state: &ZSTD_RustLazyState,
    ip: *const u8,
    iend: *const u8,
    offset: &mut usize,
    mls: u32,
    row_log: u32,
    search_method: c_int,
    dict_mode: c_int,
) -> usize {
    match search_method {
        SEARCH_HASH_CHAIN => unsafe { hc_find_best_match(state, ip, iend, offset, mls, dict_mode) },
        SEARCH_BINARY_TREE => unsafe {
            bt_find_best_match(state, ip, iend, offset, mls, dict_mode)
        },
        SEARCH_ROW_HASH => unsafe {
            row_find_best_match(state, ip, iend, offset, mls, dict_mode, row_log)
        },
        _ => 0,
    }
}

#[inline]
fn score(length: usize, multiplier: usize, offbase: usize, adjustment: usize) -> i32 {
    let raw = length
        .wrapping_mul(multiplier)
        .wrapping_sub(ZSTD_highbit32(offbase as u32) as usize)
        .wrapping_add(adjustment);
    raw as i32
}

/* ------------------------------------------------------------------------- */
/* Lazy parser for no dictionary, dictionary match state, and DDS            */
/* ------------------------------------------------------------------------- */

unsafe fn compress_block_lazy_generic(
    state: &ZSTD_RustLazyState,
    seq_store: *mut SeqStore_t,
    reps: *mut u32,
    src: *const u8,
    src_size: usize,
    search_method: c_int,
    depth: u32,
    dict_mode: c_int,
) -> usize {
    let istart = src;
    let mut ip = istart;
    let mut anchor = istart;
    let iend = istart.wrapping_add(src_size);
    let required = if search_method == SEARCH_ROW_HASH {
        HASH_READ_SIZE + ZSTD_ROW_HASH_CACHE_SIZE
    } else {
        HASH_READ_SIZE
    };
    if src_size < required {
        return src_size;
    }
    let ilimit = iend.wrapping_sub(required);
    let base = state.base;
    let prefix_lowest_index = state.dictLimit;
    let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
    let mls = bounded(4, state.minMatch, 6);
    let row_log = bounded(4, state.searchLog, 6);

    let mut offset_1 = unsafe { *reps };
    let mut offset_2 = unsafe { *reps.add(1) };
    let mut offset_saved_1 = 0u32;
    let mut offset_saved_2 = 0u32;

    let is_dms = dict_mode == DICT_MATCH_STATE;
    let is_dds = dict_mode == DICT_DEDICATED;
    let is_dxs = is_dms || is_dds;
    let dms = if is_dxs {
        Some(unsafe { dict_state(state) })
    } else {
        None
    };
    let dict_lowest_index = dms.map_or(0, |value| value.dictLimit);
    let dict_base = dms.map_or(ptr::null(), |value| value.base);
    let dict_lowest = if is_dxs {
        dict_base.wrapping_add(dict_lowest_index as usize)
    } else {
        ptr::null()
    };
    let dict_end = dms.map_or(ptr::null(), |value| value.nextSrc);
    let dict_index_delta = if is_dxs {
        prefix_lowest_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) })
    } else {
        0
    };
    let dict_and_prefix_length = if is_dxs {
        (ip as usize)
            .wrapping_sub(prefix_lowest as usize)
            .wrapping_add((dict_end as usize).wrapping_sub(dict_lowest as usize)) as u32
    } else {
        (ip as usize).wrapping_sub(prefix_lowest as usize) as u32
    };

    if dict_and_prefix_length == 0 {
        ip = ip.wrapping_add(1);
    }
    if dict_mode == DICT_NO_DICT {
        let curr = unsafe { index_from(base, ip) };
        let window_low =
            lowest_prefix_index(state.dictLimit, state.loadedDictEnd, curr, state.windowLog);
        let max_rep = curr.wrapping_sub(window_low);
        if offset_2 > max_rep {
            offset_saved_2 = offset_2;
            offset_2 = 0;
        }
        if offset_1 > max_rep {
            offset_saved_1 = offset_1;
            offset_1 = 0;
        }
    }

    unsafe { set_lazy_skipping(state, false) };
    if search_method == SEARCH_ROW_HASH {
        unsafe { row_fill_hash_cache(state, base, row_log, mls, next_to_update(state), ilimit) };
    }

    while ptr_lt(ip, ilimit) {
        let mut match_length = 0usize;
        let mut offbase = REPCODE1_TO_OFFBASE as usize;
        let mut start = ip.wrapping_add(1);
        let mut store_directly = false;

        if is_dxs {
            let rep_index = unsafe { index_from(base, ip) }
                .wrapping_add(1)
                .wrapping_sub(offset_1);
            let rep_match = if rep_index < prefix_lowest_index {
                dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
            } else {
                base.wrapping_add(rep_index as usize)
            };
            if index_overlap_check(prefix_lowest_index, rep_index)
                && unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
            {
                let rep_end = if rep_index < prefix_lowest_index {
                    dict_end
                } else {
                    iend
                };
                match_length = unsafe {
                    count_2segments(
                        ip.wrapping_add(5),
                        rep_match.wrapping_add(4),
                        iend,
                        rep_end,
                        prefix_lowest,
                    ) + 4
                };
                if depth == 0 {
                    store_directly = true;
                }
            }
        }
        if dict_mode == DICT_NO_DICT
            && offset_1 > 0
            && unsafe {
                read32(ip.wrapping_add(1).wrapping_sub(offset_1 as usize))
                    == read32(ip.wrapping_add(1))
            }
        {
            match_length = unsafe {
                count(
                    ip.wrapping_add(5),
                    ip.wrapping_add(5).wrapping_sub(offset_1 as usize),
                    iend,
                ) + 4
            };
            if depth == 0 {
                store_directly = true;
            }
        }

        if !store_directly {
            let mut candidate = 999_999_999usize;
            let found = unsafe {
                search_max(
                    state,
                    ip,
                    iend,
                    &mut candidate,
                    mls,
                    row_log,
                    search_method,
                    dict_mode,
                )
            };
            if found > match_length {
                match_length = found;
                start = ip;
                offbase = candidate;
            }

            if match_length < 4 {
                let step = (ip as usize).wrapping_sub(anchor as usize) >> K_SEARCH_STRENGTH;
                ip = ip.wrapping_add(step.wrapping_add(1));
                unsafe { set_lazy_skipping(state, step.wrapping_add(1) > K_LAZY_SKIPPING_STEP) };
                continue;
            }

            if depth >= 1 {
                while ptr_lt(ip, ilimit) {
                    ip = ip.wrapping_add(1);
                    if dict_mode == DICT_NO_DICT
                        && offbase != 0
                        && offset_1 > 0
                        && unsafe { read32(ip) == read32(ip.wrapping_sub(offset_1 as usize)) }
                    {
                        let ml_rep = unsafe {
                            count(
                                ip.wrapping_add(4),
                                ip.wrapping_add(4).wrapping_sub(offset_1 as usize),
                                iend,
                            ) + 4
                        };
                        if ml_rep >= 4
                            && score(ml_rep, 3, REPCODE1_TO_OFFBASE as usize, 0)
                                > score(match_length, 3, offbase, 1)
                        {
                            match_length = ml_rep;
                            offbase = REPCODE1_TO_OFFBASE as usize;
                            start = ip;
                        }
                    }
                    if is_dxs {
                        let rep_index = unsafe { index_from(base, ip) }.wrapping_sub(offset_1);
                        let rep_match = if rep_index < prefix_lowest_index {
                            dict_base
                                .wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
                        } else {
                            base.wrapping_add(rep_index as usize)
                        };
                        if index_overlap_check(prefix_lowest_index, rep_index)
                            && unsafe { read32(rep_match) == read32(ip) }
                        {
                            let rep_end = if rep_index < prefix_lowest_index {
                                dict_end
                            } else {
                                iend
                            };
                            let ml_rep = unsafe {
                                count_2segments(
                                    ip.wrapping_add(4),
                                    rep_match.wrapping_add(4),
                                    iend,
                                    rep_end,
                                    prefix_lowest,
                                ) + 4
                            };
                            if ml_rep >= 4
                                && score(ml_rep, 3, REPCODE1_TO_OFFBASE as usize, 0)
                                    > score(match_length, 3, offbase, 1)
                            {
                                match_length = ml_rep;
                                offbase = REPCODE1_TO_OFFBASE as usize;
                                start = ip;
                            }
                        }
                    }

                    let mut candidate = 999_999_999usize;
                    let found = unsafe {
                        search_max(
                            state,
                            ip,
                            iend,
                            &mut candidate,
                            mls,
                            row_log,
                            search_method,
                            dict_mode,
                        )
                    };
                    if found >= 4
                        && score(found, 4, candidate, 0) > score(match_length, 4, offbase, 4)
                    {
                        match_length = found;
                        offbase = candidate;
                        start = ip;
                        continue;
                    }

                    if depth == 2 && ptr_lt(ip, ilimit) {
                        ip = ip.wrapping_add(1);
                        if dict_mode == DICT_NO_DICT
                            && offbase != 0
                            && offset_1 > 0
                            && unsafe { read32(ip) == read32(ip.wrapping_sub(offset_1 as usize)) }
                        {
                            let ml_rep = unsafe {
                                count(
                                    ip.wrapping_add(4),
                                    ip.wrapping_add(4).wrapping_sub(offset_1 as usize),
                                    iend,
                                ) + 4
                            };
                            if ml_rep >= 4
                                && score(ml_rep, 4, REPCODE1_TO_OFFBASE as usize, 0)
                                    > score(match_length, 4, offbase, 1)
                            {
                                match_length = ml_rep;
                                offbase = REPCODE1_TO_OFFBASE as usize;
                                start = ip;
                            }
                        }
                        if is_dxs {
                            let rep_index = unsafe { index_from(base, ip) }.wrapping_sub(offset_1);
                            let rep_match = if rep_index < prefix_lowest_index {
                                dict_base
                                    .wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
                            } else {
                                base.wrapping_add(rep_index as usize)
                            };
                            if index_overlap_check(prefix_lowest_index, rep_index)
                                && unsafe { read32(rep_match) == read32(ip) }
                            {
                                let rep_end = if rep_index < prefix_lowest_index {
                                    dict_end
                                } else {
                                    iend
                                };
                                let ml_rep = unsafe {
                                    count_2segments(
                                        ip.wrapping_add(4),
                                        rep_match.wrapping_add(4),
                                        iend,
                                        rep_end,
                                        prefix_lowest,
                                    ) + 4
                                };
                                if ml_rep >= 4
                                    && score(ml_rep, 4, REPCODE1_TO_OFFBASE as usize, 0)
                                        > score(match_length, 4, offbase, 1)
                                {
                                    match_length = ml_rep;
                                    offbase = REPCODE1_TO_OFFBASE as usize;
                                    start = ip;
                                }
                            }
                        }

                        let mut second_candidate = 999_999_999usize;
                        let second_found = unsafe {
                            search_max(
                                state,
                                ip,
                                iend,
                                &mut second_candidate,
                                mls,
                                row_log,
                                search_method,
                                dict_mode,
                            )
                        };
                        if second_found >= 4
                            && score(second_found, 4, second_candidate, 0)
                                > score(match_length, 4, offbase, 7)
                        {
                            match_length = second_found;
                            offbase = second_candidate;
                            start = ip;
                            continue;
                        }
                    }
                    break;
                }
            }

            if offbase_is_offset(offbase) {
                let offset = offbase_to_offset(offbase);
                if dict_mode == DICT_NO_DICT {
                    while ptr_gt(start, anchor)
                        && ptr_gt(start.wrapping_sub(offset as usize), prefix_lowest)
                        && unsafe {
                            *start.wrapping_sub(1) == *start.wrapping_sub(offset as usize + 1)
                        }
                    {
                        start = start.wrapping_sub(1);
                        match_length += 1;
                    }
                }
                if is_dxs {
                    let match_index = unsafe { index_from(base, start) }.wrapping_sub(offset);
                    let mut matched = if match_index < prefix_lowest_index {
                        dict_base.wrapping_add(match_index.wrapping_sub(dict_index_delta) as usize)
                    } else {
                        base.wrapping_add(match_index as usize)
                    };
                    let match_start = if match_index < prefix_lowest_index {
                        dict_lowest
                    } else {
                        prefix_lowest
                    };
                    while ptr_gt(start, anchor)
                        && ptr_gt(matched, match_start)
                        && unsafe { *start.wrapping_sub(1) == *matched.wrapping_sub(1) }
                    {
                        start = start.wrapping_sub(1);
                        matched = matched.wrapping_sub(1);
                        match_length += 1;
                    }
                }
                offset_2 = offset_1;
                offset_1 = offset;
            }
        }

        let lit_length = (start as usize).wrapping_sub(anchor as usize);
        unsafe {
            store_seq(
                seq_store,
                lit_length,
                anchor,
                iend,
                offbase as u32,
                match_length,
            )
        };
        anchor = start.wrapping_add(match_length);
        ip = anchor;

        if unsafe { lazy_skipping(state) } {
            if search_method == SEARCH_ROW_HASH {
                unsafe {
                    row_fill_hash_cache(state, base, row_log, mls, next_to_update(state), ilimit)
                };
            }
            unsafe { set_lazy_skipping(state, false) };
        }

        if is_dxs {
            while ptr_le(ip, ilimit) {
                let current = unsafe { index_from(base, ip) };
                let rep_index = current.wrapping_sub(offset_2);
                let rep_match = if rep_index < prefix_lowest_index {
                    dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
                } else {
                    base.wrapping_add(rep_index as usize)
                };
                if index_overlap_check(prefix_lowest_index, rep_index)
                    && unsafe { read32(rep_match) == read32(ip) }
                {
                    let rep_end = if rep_index < prefix_lowest_index {
                        dict_end
                    } else {
                        iend
                    };
                    match_length = unsafe {
                        count_2segments(
                            ip.wrapping_add(4),
                            rep_match.wrapping_add(4),
                            iend,
                            rep_end,
                            prefix_lowest,
                        ) + 4
                    };
                    std::mem::swap(&mut offset_2, &mut offset_1);
                    unsafe {
                        store_seq(
                            seq_store,
                            0,
                            anchor,
                            iend,
                            REPCODE1_TO_OFFBASE,
                            match_length,
                        )
                    };
                    ip = ip.wrapping_add(match_length);
                    anchor = ip;
                    continue;
                }
                break;
            }
        }

        if dict_mode == DICT_NO_DICT {
            while ptr_le(ip, ilimit)
                && offset_2 > 0
                && unsafe { read32(ip) == read32(ip.wrapping_sub(offset_2 as usize)) }
            {
                match_length = unsafe {
                    count(
                        ip.wrapping_add(4),
                        ip.wrapping_add(4).wrapping_sub(offset_2 as usize),
                        iend,
                    ) + 4
                };
                std::mem::swap(&mut offset_2, &mut offset_1);
                unsafe {
                    store_seq(
                        seq_store,
                        0,
                        anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        match_length,
                    )
                };
                ip = ip.wrapping_add(match_length);
                anchor = ip;
            }
        }
    }

    if offset_saved_1 != 0 && offset_1 != 0 {
        offset_saved_2 = offset_saved_1;
    }
    unsafe {
        *reps = if offset_1 != 0 {
            offset_1
        } else {
            offset_saved_1
        };
        *reps.add(1) = if offset_2 != 0 {
            offset_2
        } else {
            offset_saved_2
        };
    }
    (iend as usize).wrapping_sub(anchor as usize)
}

/* ------------------------------------------------------------------------- */
/* Lazy parser for external dictionaries                                     */
/* ------------------------------------------------------------------------- */

unsafe fn compress_block_lazy_ext_dict_generic(
    state: &ZSTD_RustLazyState,
    seq_store: *mut SeqStore_t,
    reps: *mut u32,
    src: *const u8,
    src_size: usize,
    search_method: c_int,
    depth: u32,
) -> usize {
    let istart = src;
    let mut ip = istart;
    let mut anchor = istart;
    let iend = istart.wrapping_add(src_size);
    let required = if search_method == SEARCH_ROW_HASH {
        HASH_READ_SIZE + ZSTD_ROW_HASH_CACHE_SIZE
    } else {
        HASH_READ_SIZE
    };
    if src_size < required {
        return src_size;
    }
    let ilimit = iend.wrapping_sub(required);
    let base = state.base;
    let dict_limit = state.dictLimit;
    let prefix_start = base.wrapping_add(dict_limit as usize);
    let dict_base = state.dictBase;
    let dict_end = dict_base.wrapping_add(dict_limit as usize);
    let dict_start = dict_base.wrapping_add(state.lowLimit as usize);
    let mls = bounded(4, state.minMatch, 6);
    let row_log = bounded(4, state.searchLog, 6);
    let mut offset_1 = unsafe { *reps };
    let mut offset_2 = unsafe { *reps.add(1) };

    if ip == prefix_start {
        ip = ip.wrapping_add(1);
    }
    unsafe { set_lazy_skipping(state, false) };
    if search_method == SEARCH_ROW_HASH {
        unsafe { row_fill_hash_cache(state, base, row_log, mls, next_to_update(state), ilimit) };
    }

    while ptr_lt(ip, ilimit) {
        let mut match_length = 0usize;
        let mut offbase = REPCODE1_TO_OFFBASE as usize;
        let mut start = ip.wrapping_add(1);
        let mut curr = unsafe { index_from(base, ip) };
        let mut store_directly = false;

        let window_low = lowest_match_index(
            state.lowLimit,
            state.loadedDictEnd,
            curr.wrapping_add(1),
            state.windowLog,
        );
        let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1);
        let rep_base = if rep_index < dict_limit {
            dict_base
        } else {
            base
        };
        let rep_match = rep_base.wrapping_add(rep_index as usize);
        if index_overlap_check(dict_limit, rep_index)
            && offset_1 <= curr.wrapping_add(1).wrapping_sub(window_low)
            && unsafe { read32(ip.wrapping_add(1)) == read32(rep_match) }
        {
            let rep_end = if rep_index < dict_limit {
                dict_end
            } else {
                iend
            };
            match_length = unsafe {
                count_2segments(
                    ip.wrapping_add(5),
                    rep_match.wrapping_add(4),
                    iend,
                    rep_end,
                    prefix_start,
                ) + 4
            };
            if depth == 0 {
                store_directly = true;
            }
        }

        if !store_directly {
            let mut candidate = 999_999_999usize;
            let found = unsafe {
                search_max(
                    state,
                    ip,
                    iend,
                    &mut candidate,
                    mls,
                    row_log,
                    search_method,
                    DICT_EXT,
                )
            };
            if found > match_length {
                match_length = found;
                start = ip;
                offbase = candidate;
            }
            if match_length < 4 {
                let step = (ip as usize).wrapping_sub(anchor as usize) >> K_SEARCH_STRENGTH;
                ip = ip.wrapping_add(step.wrapping_add(1));
                unsafe { set_lazy_skipping(state, step > K_LAZY_SKIPPING_STEP) };
                continue;
            }

            if depth >= 1 {
                while ptr_lt(ip, ilimit) {
                    ip = ip.wrapping_add(1);
                    curr = curr.wrapping_add(1);
                    if offbase != 0 {
                        let window_low = lowest_match_index(
                            state.lowLimit,
                            state.loadedDictEnd,
                            curr,
                            state.windowLog,
                        );
                        let rep_index = curr.wrapping_sub(offset_1);
                        let rep_base = if rep_index < dict_limit {
                            dict_base
                        } else {
                            base
                        };
                        let rep_match = rep_base.wrapping_add(rep_index as usize);
                        if index_overlap_check(dict_limit, rep_index)
                            && offset_1 <= curr.wrapping_sub(window_low)
                            && unsafe { read32(ip) == read32(rep_match) }
                        {
                            let rep_end = if rep_index < dict_limit {
                                dict_end
                            } else {
                                iend
                            };
                            let rep_length = unsafe {
                                count_2segments(
                                    ip.wrapping_add(4),
                                    rep_match.wrapping_add(4),
                                    iend,
                                    rep_end,
                                    prefix_start,
                                ) + 4
                            };
                            if rep_length >= 4
                                && score(rep_length, 3, REPCODE1_TO_OFFBASE as usize, 0)
                                    > score(match_length, 3, offbase, 1)
                            {
                                match_length = rep_length;
                                offbase = REPCODE1_TO_OFFBASE as usize;
                                start = ip;
                            }
                        }
                    }

                    let mut candidate = 999_999_999usize;
                    let found = unsafe {
                        search_max(
                            state,
                            ip,
                            iend,
                            &mut candidate,
                            mls,
                            row_log,
                            search_method,
                            DICT_EXT,
                        )
                    };
                    if found >= 4
                        && score(found, 4, candidate, 0) > score(match_length, 4, offbase, 4)
                    {
                        match_length = found;
                        offbase = candidate;
                        start = ip;
                        continue;
                    }

                    if depth == 2 && ptr_lt(ip, ilimit) {
                        ip = ip.wrapping_add(1);
                        curr = curr.wrapping_add(1);
                        if offbase != 0 {
                            let window_low = lowest_match_index(
                                state.lowLimit,
                                state.loadedDictEnd,
                                curr,
                                state.windowLog,
                            );
                            let rep_index = curr.wrapping_sub(offset_1);
                            let rep_base = if rep_index < dict_limit {
                                dict_base
                            } else {
                                base
                            };
                            let rep_match = rep_base.wrapping_add(rep_index as usize);
                            if index_overlap_check(dict_limit, rep_index)
                                && offset_1 <= curr.wrapping_sub(window_low)
                                && unsafe { read32(ip) == read32(rep_match) }
                            {
                                let rep_end = if rep_index < dict_limit {
                                    dict_end
                                } else {
                                    iend
                                };
                                let rep_length = unsafe {
                                    count_2segments(
                                        ip.wrapping_add(4),
                                        rep_match.wrapping_add(4),
                                        iend,
                                        rep_end,
                                        prefix_start,
                                    ) + 4
                                };
                                if rep_length >= 4
                                    && score(rep_length, 4, REPCODE1_TO_OFFBASE as usize, 0)
                                        > score(match_length, 4, offbase, 1)
                                {
                                    match_length = rep_length;
                                    offbase = REPCODE1_TO_OFFBASE as usize;
                                    start = ip;
                                }
                            }
                        }

                        let mut second_candidate = 999_999_999usize;
                        let second_found = unsafe {
                            search_max(
                                state,
                                ip,
                                iend,
                                &mut second_candidate,
                                mls,
                                row_log,
                                search_method,
                                DICT_EXT,
                            )
                        };
                        if second_found >= 4
                            && score(second_found, 4, second_candidate, 0)
                                > score(match_length, 4, offbase, 7)
                        {
                            match_length = second_found;
                            offbase = second_candidate;
                            start = ip;
                            continue;
                        }
                    }
                    break;
                }
            }

            if offbase_is_offset(offbase) {
                let offset = offbase_to_offset(offbase);
                let match_index = unsafe { index_from(base, start) }.wrapping_sub(offset);
                let mut matched = if match_index < dict_limit {
                    dict_base.wrapping_add(match_index as usize)
                } else {
                    base.wrapping_add(match_index as usize)
                };
                let match_start = if match_index < dict_limit {
                    dict_start
                } else {
                    prefix_start
                };
                while ptr_gt(start, anchor)
                    && ptr_gt(matched, match_start)
                    && unsafe { *start.wrapping_sub(1) == *matched.wrapping_sub(1) }
                {
                    start = start.wrapping_sub(1);
                    matched = matched.wrapping_sub(1);
                    match_length += 1;
                }
                offset_2 = offset_1;
                offset_1 = offset;
            }
        }

        let lit_length = (start as usize).wrapping_sub(anchor as usize);
        unsafe {
            store_seq(
                seq_store,
                lit_length,
                anchor,
                iend,
                offbase as u32,
                match_length,
            )
        };
        anchor = start.wrapping_add(match_length);
        ip = anchor;

        if unsafe { lazy_skipping(state) } {
            if search_method == SEARCH_ROW_HASH {
                unsafe {
                    row_fill_hash_cache(state, base, row_log, mls, next_to_update(state), ilimit)
                };
            }
            unsafe { set_lazy_skipping(state, false) };
        }

        while ptr_le(ip, ilimit) {
            let rep_current = unsafe { index_from(base, ip) };
            let window_low = lowest_match_index(
                state.lowLimit,
                state.loadedDictEnd,
                rep_current,
                state.windowLog,
            );
            let rep_index = rep_current.wrapping_sub(offset_2);
            let rep_base = if rep_index < dict_limit {
                dict_base
            } else {
                base
            };
            let rep_match = rep_base.wrapping_add(rep_index as usize);
            if index_overlap_check(dict_limit, rep_index)
                && offset_2 <= rep_current.wrapping_sub(window_low)
                && unsafe { read32(ip) == read32(rep_match) }
            {
                let rep_end = if rep_index < dict_limit {
                    dict_end
                } else {
                    iend
                };
                match_length = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        rep_match.wrapping_add(4),
                        iend,
                        rep_end,
                        prefix_start,
                    ) + 4
                };
                std::mem::swap(&mut offset_2, &mut offset_1);
                unsafe {
                    store_seq(
                        seq_store,
                        0,
                        anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        match_length,
                    )
                };
                ip = ip.wrapping_add(match_length);
                anchor = ip;
                continue;
            }
            break;
        }
    }

    unsafe {
        *reps = offset_1;
        *reps.add(1) = offset_2;
    }
    (iend as usize).wrapping_sub(anchor as usize)
}

/* ------------------------------------------------------------------------- */
/* C ABI                                                                     */
/* ------------------------------------------------------------------------- */

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_lazy_insertAndFindFirstIndex(
    state: *mut ZSTD_RustLazyState,
    ip: *const c_void,
) -> u32 {
    let state = unsafe { &*state };
    unsafe { insert_and_find_first_index_internal(state, ip.cast::<u8>(), state.minMatch, false) }
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_lazy_row_update(
    state: *mut ZSTD_RustLazyState,
    ip: *const c_void,
) {
    let state = unsafe { &*state };
    let row_log = bounded(4, state.searchLog, 6);
    let row_mask = (1u32 << row_log) - 1;
    let mls = state.minMatch.min(6);
    unsafe { row_update_internal(state, ip.cast::<u8>(), mls, row_log, row_mask, false) };
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_lazy_loadDedicatedDict(
    state: *mut ZSTD_RustLazyState,
    ip: *const c_void,
) {
    let state = unsafe { &*state };
    unsafe { dedicated_dict_search_load_dictionary(state, ip.cast::<u8>()) };
}

#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_lazy(
    state: *mut ZSTD_RustLazyState,
    seq_store: *mut c_void,
    reps: *mut u32,
    src: *const c_void,
    src_size: usize,
    search_method: c_int,
    depth: u32,
    dict_mode: c_int,
) -> usize {
    let state = unsafe { &*state };
    if dict_mode == DICT_EXT {
        unsafe {
            compress_block_lazy_ext_dict_generic(
                state,
                seq_store.cast::<SeqStore_t>(),
                reps,
                src.cast::<u8>(),
                src_size,
                search_method,
                depth,
            )
        }
    } else {
        unsafe {
            compress_block_lazy_generic(
                state,
                seq_store.cast::<SeqStore_t>(),
                reps,
                src.cast::<u8>(),
                src_size,
                search_method,
                depth,
                dict_mode,
            )
        }
    }
}