zstd-rs/rust/src/zstd_double_fast.rs

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]

//! Double-fast block match finder.
//!
//! The C shim keeps `ZSTD_MatchState_t` opaque and supplies only the leaf
//! fields used by this two-table matcher.  The long (8-byte) and short
//! (min-match) search loops, table fills, and sequence writes live here.

use crate::mem::{
    MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_read64, MEM_readLE32, MEM_readLE64,
    MEM_readST,
};
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::c_int;
use std::ptr;

const ZSTD_REP_NUM: usize = 3;
const MINMATCH: usize = 3;
const HASH_READ_SIZE: usize = 8;
const SHORT_CACHE_TAG_BITS: u32 = 8;
const SHORT_CACHE_TAG_MASK: u32 = (1 << SHORT_CACHE_TAG_BITS) - 1;
const K_SEARCH_STRENGTH: usize = 8;
const REPCODE1_TO_OFFBASE: u32 = 1;

#[repr(C)]
#[derive(Clone, Copy)]
struct SeqDef {
    offBase: u32,
    litLength: u16,
    mlBase: u16,
}

/// The C shim checks this leaf ABI. `ZSTD_MatchState_t` remains wholly in C.
#[repr(C)]
struct SeqStore_t {
    sequencesStart: *mut SeqDef,
    sequences: *mut SeqDef,
    litStart: *mut u8,
    lit: *mut u8,
    llCode: *mut u8,
    mlCode: *mut u8,
    ofCode: *mut u8,
    maxNbSeq: usize,
    maxNbLit: usize,
    longLengthType: c_int,
    longLengthPos: u32,
}

#[inline]
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
    (left as usize) < (right as usize)
}

#[inline]
fn ptr_le(left: *const u8, right: *const u8) -> bool {
    (left as usize) <= (right as usize)
}

#[inline]
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
    (left as usize) > (right as usize)
}

#[inline]
fn ptr_ge(left: *const u8, right: *const u8) -> bool {
    (left as usize) >= (right as usize)
}

#[inline]
unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 {
    unsafe { ptr.offset_from(base) as u32 }
}

#[inline]
unsafe fn read32(ptr: *const u8) -> u32 {
    unsafe { MEM_read32(ptr.cast::<c_void>()) }
}

#[inline]
unsafe fn read64(ptr: *const u8) -> u64 {
    unsafe { MEM_read64(ptr.cast::<c_void>()) }
}

#[inline]
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
    unsafe { *table.add(index) }
}

#[inline]
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
    unsafe { *table.add(index) = value };
}

#[inline]
fn hash_shift32(value: u32, hbits: u32) -> usize {
    if hbits == 0 {
        0
    } else {
        (value >> (32 - hbits)) as usize
    }
}

#[inline]
fn hash_shift64(value: u64, hbits: u32) -> usize {
    if hbits == 0 {
        0
    } else {
        (value >> (64 - hbits)) as usize
    }
}

#[inline]
unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize {
    match mls {
        5 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits)
        }
        6 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(16).wrapping_mul(227_718_039_650_203),
                hbits,
            )
        }
        7 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627),
                hbits,
            )
        }
        8 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits)
        }
        _ => {
            let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
            hash_shift32(value.wrapping_mul(2_654_435_761), hbits)
        }
    }
}

#[inline]
fn common_bytes(word: usize) -> usize {
    let zeros = if MEM_isLittleEndian() {
        word.trailing_zeros()
    } else {
        word.leading_zeros()
    };
    (zeros / 8) as usize
}

unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
    let input_start = input;
    let word_size = size_of::<usize>();
    while unsafe { input_limit.offset_from(input) as usize } >= word_size {
        let diff =
            unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
        if diff != 0 {
            return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff);
        }
        input = input.wrapping_add(word_size);
        matched = matched.wrapping_add(word_size);
    }
    if MEM_64bits()
        && unsafe { input_limit.offset_from(input) as usize } >= 4
        && unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
    {
        input = input.wrapping_add(4);
        matched = matched.wrapping_add(4);
    }
    if unsafe { input_limit.offset_from(input) as usize } >= 2
        && unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
    {
        input = input.wrapping_add(2);
        matched = matched.wrapping_add(2);
    }
    if ptr_lt(input, input_limit) && unsafe { *input == *matched } {
        input = input.wrapping_add(1);
    }
    unsafe { input.offset_from(input_start) as usize }
}

unsafe fn count_2segments(
    input: *const u8,
    matched: *const u8,
    input_end: *const u8,
    match_end: *const u8,
    input_start: *const u8,
) -> usize {
    let match_remaining = unsafe { match_end.offset_from(matched) as usize };
    let input_remaining = unsafe { input_end.offset_from(input) as usize };
    let first_end = input.wrapping_add(match_remaining.min(input_remaining));
    let first_count = unsafe { count(input, matched, first_end) };
    if matched.wrapping_add(first_count) != match_end {
        return first_count;
    }
    first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
}

#[inline]
fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
    let max_distance = 1u32.wrapping_shl(window_log);
    let within_window = if curr.wrapping_sub(dict_limit) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        dict_limit
    };
    if loaded_dict_end != 0 {
        dict_limit
    } else {
        within_window
    }
}

#[inline]
fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
    let max_distance = 1u32.wrapping_shl(window_log);
    let within_window = if curr.wrapping_sub(low_limit) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        low_limit
    };
    if loaded_dict_end != 0 {
        low_limit
    } else {
        within_window
    }
}

#[inline]
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
    /*
     * Keep the C predicate's two valid coordinate regions without relying on
     * unsigned subtraction to classify an already-invalid index.  The
     * attached-dictionary coordinate guard below additionally bounds prefix
     * repcodes by the current position.
     */
    rep_index >= prefix_lowest_index
        || rep_index
            .checked_add(4)
            .is_some_and(|rep_end| rep_end <= prefix_lowest_index)
}

#[inline]
fn attached_rep_index(
    prefix_lowest_index: u32,
    dict_start_index: u32,
    dict_index_delta: u32,
    current: u32,
    advance: u32,
    offset: u32,
) -> Option<u32> {
    if offset == 0 {
        return None;
    }
    let rep_index = current.checked_add(advance)?.checked_sub(offset)?;
    if rep_index > current || !index_overlap_check(prefix_lowest_index, rep_index) {
        return None;
    }
    if rep_index < prefix_lowest_index {
        let dict_rep_start = dict_start_index.checked_add(dict_index_delta)?;
        if rep_index < dict_rep_start {
            return None;
        }
    }
    Some(rep_index)
}

#[inline]
fn write_tagged_index(table: *mut u32, hash_and_tag: usize, index: u32) {
    let hash = hash_and_tag >> SHORT_CACHE_TAG_BITS;
    let tag = (hash_and_tag as u32) & SHORT_CACHE_TAG_MASK;
    unsafe { table_set(table, hash, (index << SHORT_CACHE_TAG_BITS) | tag) };
}

#[inline]
fn packed_tags_match(first: u32, second: usize) -> bool {
    (first & SHORT_CACHE_TAG_MASK) == ((second as u32) & SHORT_CACHE_TAG_MASK)
}

unsafe fn store_seq(
    seq_store: *mut SeqStore_t,
    lit_length: usize,
    literals: *const u8,
    _lit_limit: *const u8,
    off_base: u32,
    match_length: usize,
) {
    let seq_store = unsafe { &mut *seq_store };
    let sequence = seq_store.sequences;
    debug_assert!(
        unsafe { sequence.offset_from(seq_store.sequencesStart) as usize } < seq_store.maxNbSeq
    );
    debug_assert!(match_length >= MINMATCH);
    if lit_length != 0 {
        unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) };
    }
    seq_store.lit = seq_store.lit.wrapping_add(lit_length);

    let sequence_index = unsafe { sequence.offset_from(seq_store.sequencesStart) as u32 };
    if lit_length > u16::MAX as usize {
        debug_assert_eq!(seq_store.longLengthType, 0);
        seq_store.longLengthType = 1;
        seq_store.longLengthPos = sequence_index;
    }
    unsafe { (*sequence).litLength = lit_length as u16 };
    unsafe { (*sequence).offBase = off_base };
    let match_base = match_length - MINMATCH;
    if match_base > u16::MAX as usize {
        debug_assert_eq!(seq_store.longLengthType, 0);
        seq_store.longLengthType = 2;
        seq_store.longLengthPos = sequence_index;
    }
    unsafe { (*sequence).mlBase = match_base as u16 };
    seq_store.sequences = sequence.wrapping_add(1);
}

#[inline]
fn fast_mls(min_match: u32) -> u32 {
    match min_match {
        5..=7 => min_match,
        _ => 4,
    }
}

unsafe fn fill_double_hash_table(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    next_to_update: u32,
    end: *const u8,
    hash_log: u32,
    chain_log: u32,
    min_match: u32,
    full_table_load: bool,
    tagged_indices: bool,
) {
    if unsafe { end.offset_from(base) } < HASH_READ_SIZE as isize {
        return;
    }
    let hbits_long = hash_log
        + if tagged_indices {
            SHORT_CACHE_TAG_BITS
        } else {
            0
        };
    let hbits_small = chain_log
        + if tagged_indices {
            SHORT_CACHE_TAG_BITS
        } else {
            0
        };
    let mut input = base.wrapping_add(next_to_update as usize);
    let input_end = end.wrapping_sub(HASH_READ_SIZE);

    while ptr_le(input.wrapping_add(2), input_end) {
        let current = unsafe { index_from(base, input) };
        for position in 0..3usize {
            let small_hash =
                unsafe { hash_ptr(input.wrapping_add(position), hbits_small, min_match) };
            let long_hash = unsafe { hash_ptr(input.wrapping_add(position), hbits_long, 8) };
            let small_index = if tagged_indices {
                small_hash >> SHORT_CACHE_TAG_BITS
            } else {
                small_hash
            };
            let long_index = if tagged_indices {
                long_hash >> SHORT_CACHE_TAG_BITS
            } else {
                long_hash
            };
            if position == 0 {
                if tagged_indices {
                    write_tagged_index(
                        hash_small,
                        small_hash,
                        current.wrapping_add(position as u32),
                    );
                } else {
                    unsafe {
                        table_set(
                            hash_small,
                            small_index,
                            current.wrapping_add(position as u32),
                        )
                    };
                }
            }
            if position == 0 || unsafe { table_get(hash_long, long_index) } == 0 {
                if tagged_indices {
                    write_tagged_index(hash_long, long_hash, current.wrapping_add(position as u32));
                } else {
                    unsafe {
                        table_set(hash_long, long_index, current.wrapping_add(position as u32))
                    };
                }
            }
            if !full_table_load {
                break;
            }
        }
        input = input.wrapping_add(3);
    }
}

/// Rust implementation called by the C ABI wrapper for `ZSTD_fillDoubleHashTable`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_fillDoubleHashTable(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    next_to_update: u32,
    end: *const c_void,
    hash_log: u32,
    chain_log: u32,
    min_match: u32,
    full_table_load: c_int,
    for_cdict: c_int,
) {
    unsafe {
        fill_double_hash_table(
            hash_long,
            hash_small,
            base,
            next_to_update,
            end.cast::<u8>(),
            hash_log,
            chain_log,
            min_match,
            full_table_load != 0,
            for_cdict != 0,
        )
    };
}

unsafe fn post_no_dict_match(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    hbits_long: u32,
    hbits_small: u32,
    mls: u32,
    seq_store: *mut SeqStore_t,
    offset1: &mut u32,
    offset2: &mut u32,
    mut ip: *const u8,
    match_length: usize,
    current: u32,
    iend: *const u8,
    ilimit: *const u8,
) -> (*const u8, *const u8) {
    ip = ip.wrapping_add(match_length);
    let mut anchor = ip;
    if ptr_le(ip, ilimit) {
        let index_to_insert = current.wrapping_add(2);
        let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
        unsafe { table_set(hash_long, hash, index_to_insert) };
        let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
        unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
        let hash = unsafe {
            hash_ptr(
                base.wrapping_add(index_to_insert as usize),
                hbits_small,
                mls,
            )
        };
        unsafe { table_set(hash_small, hash, index_to_insert) };
        let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
        unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };

        while ptr_le(ip, ilimit)
            && *offset2 > 0
            && unsafe { read32(ip) == read32(ip.wrapping_sub(*offset2 as usize)) }
        {
            let repeat_length = unsafe {
                count(
                    ip.wrapping_add(4),
                    ip.wrapping_add(4).wrapping_sub(*offset2 as usize),
                    iend,
                ) + 4
            };
            std::mem::swap(offset1, offset2);
            let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
            unsafe { table_set(hash_small, hash, index_from(base, ip)) };
            let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
            unsafe { table_set(hash_long, hash, index_from(base, ip)) };
            unsafe {
                store_seq(
                    seq_store,
                    0,
                    anchor,
                    iend,
                    REPCODE1_TO_OFFBASE,
                    repeat_length,
                )
            };
            ip = ip.wrapping_add(repeat_length);
            anchor = ip;
        }
    }
    (ip, anchor)
}

unsafe fn compress_block_double_fast_no_dict(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    dict_limit: u32,
    loaded_dict_end: u32,
    hash_log: u32,
    chain_log: u32,
    window_log: u32,
    seq_store: *mut SeqStore_t,
    reps: *mut u32,
    src: *const u8,
    src_size: usize,
    mls: u32,
) -> usize {
    if src_size < HASH_READ_SIZE {
        return src_size;
    }
    let istart = src;
    let iend = istart.wrapping_add(src_size);
    let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
    let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
    let prefix_lowest_index =
        lowest_prefix_index(dict_limit, loaded_dict_end, end_index, window_log);
    let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
    let mut ip = istart;
    let mut anchor = istart;
    let mut offset1 = unsafe { *reps };
    let mut offset2 = unsafe { *reps.add(1) };
    let mut offset_saved1 = 0u32;
    let mut offset_saved2 = 0u32;

    if ip == prefix_lowest {
        ip = ip.wrapping_add(1);
    }
    let current = unsafe { index_from(base, ip) };
    let window_low = lowest_prefix_index(dict_limit, loaded_dict_end, current, window_log);
    let max_rep = current.wrapping_sub(window_low);
    if offset2 > max_rep {
        offset_saved2 = offset2;
        offset2 = 0;
    }
    if offset1 > max_rep {
        offset_saved1 = offset1;
        offset1 = 0;
    }

    'outer: loop {
        let mut step = 1usize;
        let mut next_step = ip.wrapping_add(1 << K_SEARCH_STRENGTH);
        let mut ip1 = ip.wrapping_add(step);
        if ptr_gt(ip1, ilimit) {
            break;
        }

        let mut hash_long0 = unsafe { hash_ptr(ip, hash_log, 8) };
        let mut index_long0 = unsafe { table_get(hash_long, hash_long0) };
        let mut match_long0 = base.wrapping_add(index_long0 as usize);

        loop {
            let hash_small0 = unsafe { hash_ptr(ip, chain_log, mls) };
            let index_small0 = unsafe { table_get(hash_small, hash_small0) };
            let mut match_small0 = base.wrapping_add(index_small0 as usize);
            let current = unsafe { index_from(base, ip) };
            unsafe {
                table_set(hash_long, hash_long0, current);
                table_set(hash_small, hash_small0, current);
            }

            if offset1 > 0
                && unsafe {
                    read32(ip.wrapping_add(1).wrapping_sub(offset1 as usize))
                        == read32(ip.wrapping_add(1))
                }
            {
                let match_length = unsafe {
                    count(
                        ip.wrapping_add(5),
                        ip.wrapping_add(5).wrapping_sub(offset1 as usize),
                        iend,
                    ) + 4
                };
                ip = ip.wrapping_add(1);
                unsafe {
                    store_seq(
                        seq_store,
                        ip.offset_from(anchor) as usize,
                        anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        match_length,
                    )
                };
                let (next_ip, next_anchor) = unsafe {
                    post_no_dict_match(
                        hash_long,
                        hash_small,
                        base,
                        hash_log,
                        chain_log,
                        mls,
                        seq_store,
                        &mut offset1,
                        &mut offset2,
                        ip,
                        match_length,
                        current,
                        iend,
                        ilimit,
                    )
                };
                ip = next_ip;
                anchor = next_anchor;
                continue 'outer;
            }

            let hash_long1 = unsafe { hash_ptr(ip1, hash_log, 8) };
            if index_long0 > prefix_lowest_index && unsafe { read64(match_long0) == read64(ip) } {
                let mut match_length =
                    unsafe { count(ip.wrapping_add(8), match_long0.wrapping_add(8), iend) + 8 };
                let offset = unsafe { index_from(match_long0, ip) };
                while ptr_gt(ip, anchor)
                    && ptr_gt(match_long0, prefix_lowest)
                    && unsafe { *ip.wrapping_sub(1) == *match_long0.wrapping_sub(1) }
                {
                    ip = ip.wrapping_sub(1);
                    match_long0 = match_long0.wrapping_sub(1);
                    match_length += 1;
                }
                offset2 = offset1;
                offset1 = offset;
                if step < 4 {
                    unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) };
                }
                unsafe {
                    store_seq(
                        seq_store,
                        ip.offset_from(anchor) as usize,
                        anchor,
                        iend,
                        offset.wrapping_add(ZSTD_REP_NUM as u32),
                        match_length,
                    )
                };
                let (next_ip, next_anchor) = unsafe {
                    post_no_dict_match(
                        hash_long,
                        hash_small,
                        base,
                        hash_log,
                        chain_log,
                        mls,
                        seq_store,
                        &mut offset1,
                        &mut offset2,
                        ip,
                        match_length,
                        current,
                        iend,
                        ilimit,
                    )
                };
                ip = next_ip;
                anchor = next_anchor;
                continue 'outer;
            }

            let index_long1 = unsafe { table_get(hash_long, hash_long1) };
            let match_long1 = base.wrapping_add(index_long1 as usize);
            if index_small0 > prefix_lowest_index && unsafe { read32(match_small0) == read32(ip) } {
                let mut match_length =
                    unsafe { count(ip.wrapping_add(4), match_small0.wrapping_add(4), iend) + 4 };
                let mut offset = unsafe { index_from(match_small0, ip) };
                if index_long1 > prefix_lowest_index
                    && unsafe { read64(match_long1) == read64(ip1) }
                {
                    let long_length = unsafe {
                        count(ip1.wrapping_add(8), match_long1.wrapping_add(8), iend) + 8
                    };
                    if long_length > match_length {
                        ip = ip1;
                        match_length = long_length;
                        offset = unsafe { index_from(match_long1, ip) };
                        match_small0 = match_long1;
                    }
                }
                while ptr_gt(ip, anchor)
                    && ptr_gt(match_small0, prefix_lowest)
                    && unsafe { *ip.wrapping_sub(1) == *match_small0.wrapping_sub(1) }
                {
                    ip = ip.wrapping_sub(1);
                    match_small0 = match_small0.wrapping_sub(1);
                    match_length += 1;
                }
                offset2 = offset1;
                offset1 = offset;
                if step < 4 {
                    unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) };
                }
                unsafe {
                    store_seq(
                        seq_store,
                        ip.offset_from(anchor) as usize,
                        anchor,
                        iend,
                        offset.wrapping_add(ZSTD_REP_NUM as u32),
                        match_length,
                    )
                };
                let (next_ip, next_anchor) = unsafe {
                    post_no_dict_match(
                        hash_long,
                        hash_small,
                        base,
                        hash_log,
                        chain_log,
                        mls,
                        seq_store,
                        &mut offset1,
                        &mut offset2,
                        ip,
                        match_length,
                        current,
                        iend,
                        ilimit,
                    )
                };
                ip = next_ip;
                anchor = next_anchor;
                continue 'outer;
            }

            if ptr_ge(ip1, next_step) {
                step += 1;
                next_step = next_step.wrapping_add(1 << K_SEARCH_STRENGTH);
            }
            ip = ip1;
            ip1 = ip1.wrapping_add(step);
            hash_long0 = hash_long1;
            index_long0 = index_long1;
            match_long0 = match_long1;
            if ptr_gt(ip1, ilimit) {
                break 'outer;
            }
        }
    }

    offset_saved2 = if offset_saved1 != 0 && offset1 != 0 {
        offset_saved1
    } else {
        offset_saved2
    };
    unsafe {
        *reps = if offset1 != 0 { offset1 } else { offset_saved1 };
        *reps.add(1) = if offset2 != 0 { offset2 } else { offset_saved2 };
    }
    unsafe { iend.offset_from(anchor) as usize }
}

/// Rust implementation called by the C ABI wrapper for `ZSTD_compressBlock_doubleFast`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    dict_limit: u32,
    loaded_dict_end: u32,
    hash_log: u32,
    chain_log: u32,
    min_match: u32,
    window_log: u32,
    seq_store: *mut c_void,
    reps: *mut u32,
    src: *const c_void,
    src_size: usize,
) -> usize {
    unsafe {
        compress_block_double_fast_no_dict(
            hash_long,
            hash_small,
            base,
            dict_limit,
            loaded_dict_end,
            hash_log,
            chain_log,
            window_log,
            seq_store.cast::<SeqStore_t>(),
            reps,
            src.cast::<u8>(),
            src_size,
            fast_mls(min_match),
        )
    }
}

unsafe fn post_dict_match(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    dict_base: *const u8,
    prefix_lowest_index: u32,
    dict_start_index: u32,
    dict_index_delta: u32,
    dict_end: *const u8,
    prefix_lowest: *const u8,
    hbits_long: u32,
    hbits_small: u32,
    mls: u32,
    seq_store: *mut SeqStore_t,
    offset1: &mut u32,
    offset2: &mut u32,
    mut ip: *const u8,
    match_length: usize,
    current: u32,
    iend: *const u8,
    ilimit: *const u8,
) -> (*const u8, *const u8) {
    ip = ip.wrapping_add(match_length);
    let mut anchor = ip;
    if ptr_le(ip, ilimit) {
        let index_to_insert = current.wrapping_add(2);
        let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
        unsafe { table_set(hash_long, hash, index_to_insert) };
        let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
        unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
        let hash = unsafe {
            hash_ptr(
                base.wrapping_add(index_to_insert as usize),
                hbits_small,
                mls,
            )
        };
        unsafe { table_set(hash_small, hash, index_to_insert) };
        let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
        unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };

        while ptr_le(ip, ilimit) {
            let current2 = unsafe { index_from(base, ip) };
            let Some(rep_index2) = attached_rep_index(
                prefix_lowest_index,
                dict_start_index,
                dict_index_delta,
                current2,
                0,
                *offset2,
            ) else {
                break;
            };
            let rep_match2 = if rep_index2 < prefix_lowest_index {
                dict_base.wrapping_add((rep_index2 - dict_index_delta) as usize)
            } else {
                base.wrapping_add(rep_index2 as usize)
            };
            if unsafe { read32(rep_match2) == read32(ip) } {
                let rep_end2 = if rep_index2 < prefix_lowest_index {
                    dict_end
                } else {
                    iend
                };
                let repeat_length = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        rep_match2.wrapping_add(4),
                        iend,
                        rep_end2,
                        prefix_lowest,
                    ) + 4
                };
                std::mem::swap(offset1, offset2);
                unsafe {
                    store_seq(
                        seq_store,
                        0,
                        anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        repeat_length,
                    )
                };
                let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
                unsafe { table_set(hash_small, hash, current2) };
                let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
                unsafe { table_set(hash_long, hash, current2) };
                ip = ip.wrapping_add(repeat_length);
                anchor = ip;
                continue;
            }
            break;
        }
    }
    (ip, anchor)
}

unsafe fn compress_block_double_fast_dict_match_state(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    prefix_lowest_index: u32,
    hash_log: u32,
    chain_log: u32,
    seq_store: *mut SeqStore_t,
    reps: *mut u32,
    src: *const u8,
    src_size: usize,
    mls: u32,
    dict_hash_long: *const u32,
    dict_hash_small: *const u32,
    dict_base: *const u8,
    dict_start_index: u32,
    dict_end: *const u8,
    dict_hash_log: u32,
    dict_chain_log: u32,
    _prefetch_cdict_tables: bool,
) -> usize {
    if src_size < HASH_READ_SIZE {
        return src_size;
    }
    let istart = src;
    let iend = istart.wrapping_add(src_size);
    let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
    let mut ip = istart;
    let mut anchor = istart;
    let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
    let dict_start = dict_base.wrapping_add(dict_start_index as usize);
    let dict_index_delta =
        prefix_lowest_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) });
    let dict_hbits_long = dict_hash_log + SHORT_CACHE_TAG_BITS;
    let dict_hbits_small = dict_chain_log + SHORT_CACHE_TAG_BITS;
    let dict_and_prefix_length =
        unsafe { ip.offset_from(prefix_lowest) as u32 + dict_end.offset_from(dict_start) as u32 };
    let mut offset1 = unsafe { *reps };
    let mut offset2 = unsafe { *reps.add(1) };
    if dict_and_prefix_length == 0 {
        ip = ip.wrapping_add(1);
    }

    'outer: while ptr_lt(ip, ilimit) {
        let h_long = unsafe { hash_ptr(ip, hash_log, 8) };
        let h_small = unsafe { hash_ptr(ip, chain_log, mls) };
        let dict_hash_and_tag_long = unsafe { hash_ptr(ip, dict_hbits_long, 8) };
        let dict_hash_and_tag_small = unsafe { hash_ptr(ip, dict_hbits_small, mls) };
        let dict_match_index_and_tag_long = unsafe {
            table_get(
                dict_hash_long,
                dict_hash_and_tag_long >> SHORT_CACHE_TAG_BITS,
            )
        };
        let dict_match_index_and_tag_small = unsafe {
            table_get(
                dict_hash_small,
                dict_hash_and_tag_small >> SHORT_CACHE_TAG_BITS,
            )
        };
        let dict_tags_match_long =
            packed_tags_match(dict_match_index_and_tag_long, dict_hash_and_tag_long);
        let dict_tags_match_small =
            packed_tags_match(dict_match_index_and_tag_small, dict_hash_and_tag_small);
        let current = unsafe { index_from(base, ip) };
        let match_index_long = unsafe { table_get(hash_long, h_long) };
        let mut match_index_small = unsafe { table_get(hash_small, h_small) };
        let mut match_long = base.wrapping_add(match_index_long as usize);
        let mut matched = base.wrapping_add(match_index_small as usize);
        unsafe {
            table_set(hash_long, h_long, current);
            table_set(hash_small, h_small, current);
        }

        if let Some(rep_index) = attached_rep_index(
            prefix_lowest_index,
            dict_start_index,
            dict_index_delta,
            current,
            1,
            offset1,
        ) {
            let rep_match = if rep_index < prefix_lowest_index {
                dict_base.wrapping_add((rep_index - dict_index_delta) as usize)
            } else {
                base.wrapping_add(rep_index as usize)
            };
            if unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) } {
                let rep_match_end = if rep_index < prefix_lowest_index {
                    dict_end
                } else {
                    iend
                };
                let match_length = unsafe {
                    count_2segments(
                        ip.wrapping_add(5),
                        rep_match.wrapping_add(4),
                        iend,
                        rep_match_end,
                        prefix_lowest,
                    ) + 4
                };
                ip = ip.wrapping_add(1);
                unsafe {
                    store_seq(
                        seq_store,
                        ip.offset_from(anchor) as usize,
                        anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        match_length,
                    )
                };
                let (next_ip, next_anchor) = unsafe {
                    post_dict_match(
                        hash_long,
                        hash_small,
                        base,
                        dict_base,
                        prefix_lowest_index,
                        dict_start_index,
                        dict_index_delta,
                        dict_end,
                        prefix_lowest,
                        hash_log,
                        chain_log,
                        mls,
                        seq_store,
                        &mut offset1,
                        &mut offset2,
                        ip,
                        match_length,
                        current,
                        iend,
                        ilimit,
                    )
                };
                ip = next_ip;
                anchor = next_anchor;
                continue;
            }
        }

        if match_index_long >= prefix_lowest_index && unsafe { read64(match_long) == read64(ip) } {
            let mut match_length =
                unsafe { count(ip.wrapping_add(8), match_long.wrapping_add(8), iend) + 8 };
            let offset = unsafe { index_from(match_long, ip) };
            while ptr_gt(ip, anchor)
                && ptr_gt(match_long, prefix_lowest)
                && unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) }
            {
                ip = ip.wrapping_sub(1);
                match_long = match_long.wrapping_sub(1);
                match_length += 1;
            }
            offset2 = offset1;
            offset1 = offset;
            unsafe {
                store_seq(
                    seq_store,
                    ip.offset_from(anchor) as usize,
                    anchor,
                    iend,
                    offset.wrapping_add(ZSTD_REP_NUM as u32),
                    match_length,
                )
            };
            let (next_ip, next_anchor) = unsafe {
                post_dict_match(
                    hash_long,
                    hash_small,
                    base,
                    dict_base,
                    prefix_lowest_index,
                    dict_start_index,
                    dict_index_delta,
                    dict_end,
                    prefix_lowest,
                    hash_log,
                    chain_log,
                    mls,
                    seq_store,
                    &mut offset1,
                    &mut offset2,
                    ip,
                    match_length,
                    current,
                    iend,
                    ilimit,
                )
            };
            ip = next_ip;
            anchor = next_anchor;
            continue;
        }

        if dict_tags_match_long {
            let dict_match_index = dict_match_index_and_tag_long >> SHORT_CACHE_TAG_BITS;
            let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
            if ptr_gt(dict_match, dict_start) && unsafe { read64(dict_match) == read64(ip) } {
                let mut match_length = unsafe {
                    count_2segments(
                        ip.wrapping_add(8),
                        dict_match.wrapping_add(8),
                        iend,
                        dict_end,
                        prefix_lowest,
                    ) + 8
                };
                let offset = current
                    .wrapping_sub(dict_match_index)
                    .wrapping_sub(dict_index_delta);
                while ptr_gt(ip, anchor)
                    && ptr_gt(dict_match, dict_start)
                    && unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
                {
                    ip = ip.wrapping_sub(1);
                    dict_match = dict_match.wrapping_sub(1);
                    match_length += 1;
                }
                offset2 = offset1;
                offset1 = offset;
                unsafe {
                    store_seq(
                        seq_store,
                        ip.offset_from(anchor) as usize,
                        anchor,
                        iend,
                        offset.wrapping_add(ZSTD_REP_NUM as u32),
                        match_length,
                    )
                };
                let (next_ip, next_anchor) = unsafe {
                    post_dict_match(
                        hash_long,
                        hash_small,
                        base,
                        dict_base,
                        prefix_lowest_index,
                        dict_start_index,
                        dict_index_delta,
                        dict_end,
                        prefix_lowest,
                        hash_log,
                        chain_log,
                        mls,
                        seq_store,
                        &mut offset1,
                        &mut offset2,
                        ip,
                        match_length,
                        current,
                        iend,
                        ilimit,
                    )
                };
                ip = next_ip;
                anchor = next_anchor;
                continue;
            }
        }

        let mut short_found = false;
        if match_index_small > prefix_lowest_index {
            short_found = unsafe { read32(matched) == read32(ip) };
        } else if dict_tags_match_small {
            let dict_match_index = dict_match_index_and_tag_small >> SHORT_CACHE_TAG_BITS;
            let dict_match = dict_base.wrapping_add(dict_match_index as usize);
            match_index_small = dict_match_index.wrapping_add(dict_index_delta);
            if ptr_gt(dict_match, dict_start) && unsafe { read32(dict_match) == read32(ip) } {
                matched = dict_match;
                short_found = true;
            }
        }
        if !short_found {
            ip = ip.wrapping_add(
                (unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1,
            );
            continue;
        }

        let hash_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) };
        let dict_hash_and_tag_long3 = unsafe { hash_ptr(ip.wrapping_add(1), dict_hbits_long, 8) };
        let match_index_long3 = unsafe { table_get(hash_long, hash_long3) };
        let dict_match_index_and_tag_long3 = unsafe {
            table_get(
                dict_hash_long,
                dict_hash_and_tag_long3 >> SHORT_CACHE_TAG_BITS,
            )
        };
        let dict_tags_match_long3 =
            packed_tags_match(dict_match_index_and_tag_long3, dict_hash_and_tag_long3);
        let mut match_long3 = base.wrapping_add(match_index_long3 as usize);
        unsafe { table_set(hash_long, hash_long3, current.wrapping_add(1)) };

        let (match_length, offset) = if match_index_long3 >= prefix_lowest_index
            && unsafe { read64(match_long3) == read64(ip.wrapping_add(1)) }
        {
            let mut length =
                unsafe { count(ip.wrapping_add(9), match_long3.wrapping_add(8), iend) + 8 };
            ip = ip.wrapping_add(1);
            let offset = unsafe { index_from(match_long3, ip) };
            while ptr_gt(ip, anchor)
                && ptr_gt(match_long3, prefix_lowest)
                && unsafe { *ip.wrapping_sub(1) == *match_long3.wrapping_sub(1) }
            {
                ip = ip.wrapping_sub(1);
                match_long3 = match_long3.wrapping_sub(1);
                length += 1;
            }
            (length, offset)
        } else if dict_tags_match_long3 {
            let dict_match_index = dict_match_index_and_tag_long3 >> SHORT_CACHE_TAG_BITS;
            let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
            if ptr_gt(dict_match, dict_start)
                && unsafe { read64(dict_match) == read64(ip.wrapping_add(1)) }
            {
                let mut length = unsafe {
                    count_2segments(
                        ip.wrapping_add(9),
                        dict_match.wrapping_add(8),
                        iend,
                        dict_end,
                        prefix_lowest,
                    ) + 8
                };
                ip = ip.wrapping_add(1);
                let offset = current
                    .wrapping_add(1)
                    .wrapping_sub(dict_match_index)
                    .wrapping_sub(dict_index_delta);
                while ptr_gt(ip, anchor)
                    && ptr_gt(dict_match, dict_start)
                    && unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
                {
                    ip = ip.wrapping_sub(1);
                    dict_match = dict_match.wrapping_sub(1);
                    length += 1;
                }
                (length, offset)
            } else if match_index_small < prefix_lowest_index {
                let mut length = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        matched.wrapping_add(4),
                        iend,
                        dict_end,
                        prefix_lowest,
                    ) + 4
                };
                let offset = current.wrapping_sub(match_index_small);
                while ptr_gt(ip, anchor)
                    && ptr_gt(matched, dict_start)
                    && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
                {
                    ip = ip.wrapping_sub(1);
                    matched = matched.wrapping_sub(1);
                    length += 1;
                }
                (length, offset)
            } else {
                let mut length =
                    unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 };
                let offset = unsafe { index_from(matched, ip) };
                while ptr_gt(ip, anchor)
                    && ptr_gt(matched, prefix_lowest)
                    && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
                {
                    ip = ip.wrapping_sub(1);
                    matched = matched.wrapping_sub(1);
                    length += 1;
                }
                (length, offset)
            }
        } else if match_index_small < prefix_lowest_index {
            let mut length = unsafe {
                count_2segments(
                    ip.wrapping_add(4),
                    matched.wrapping_add(4),
                    iend,
                    dict_end,
                    prefix_lowest,
                ) + 4
            };
            let offset = current.wrapping_sub(match_index_small);
            while ptr_gt(ip, anchor)
                && ptr_gt(matched, dict_start)
                && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
            {
                ip = ip.wrapping_sub(1);
                matched = matched.wrapping_sub(1);
                length += 1;
            }
            (length, offset)
        } else {
            let mut length =
                unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 };
            let offset = unsafe { index_from(matched, ip) };
            while ptr_gt(ip, anchor)
                && ptr_gt(matched, prefix_lowest)
                && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
            {
                ip = ip.wrapping_sub(1);
                matched = matched.wrapping_sub(1);
                length += 1;
            }
            (length, offset)
        };
        offset2 = offset1;
        offset1 = offset;
        unsafe {
            store_seq(
                seq_store,
                ip.offset_from(anchor) as usize,
                anchor,
                iend,
                offset.wrapping_add(ZSTD_REP_NUM as u32),
                match_length,
            )
        };
        let (next_ip, next_anchor) = unsafe {
            post_dict_match(
                hash_long,
                hash_small,
                base,
                dict_base,
                prefix_lowest_index,
                dict_start_index,
                dict_index_delta,
                dict_end,
                prefix_lowest,
                hash_log,
                chain_log,
                mls,
                seq_store,
                &mut offset1,
                &mut offset2,
                ip,
                match_length,
                current,
                iend,
                ilimit,
            )
        };
        ip = next_ip;
        anchor = next_anchor;
        continue 'outer;
    }

    unsafe {
        *reps = offset1;
        *reps.add(1) = offset2;
    }
    unsafe { iend.offset_from(anchor) as usize }
}

/// Rust implementation called by the C ABI wrapper for attached dictionaries.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_dictMatchState(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    prefix_lowest_index: u32,
    hash_log: u32,
    chain_log: u32,
    min_match: u32,
    seq_store: *mut c_void,
    reps: *mut u32,
    src: *const c_void,
    src_size: usize,
    dict_hash_long: *const u32,
    dict_hash_small: *const u32,
    dict_base: *const u8,
    dict_start_index: u32,
    dict_end: *const u8,
    dict_hash_log: u32,
    dict_chain_log: u32,
    prefetch_cdict_tables: c_int,
) -> usize {
    unsafe {
        compress_block_double_fast_dict_match_state(
            hash_long,
            hash_small,
            base,
            prefix_lowest_index,
            hash_log,
            chain_log,
            seq_store.cast::<SeqStore_t>(),
            reps,
            src.cast::<u8>(),
            src_size,
            fast_mls(min_match),
            dict_hash_long,
            dict_hash_small,
            dict_base,
            dict_start_index,
            dict_end,
            dict_hash_log,
            dict_chain_log,
            prefetch_cdict_tables != 0,
        )
    }
}

unsafe fn post_ext_match(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    dict_base: *const u8,
    dict_start_index: u32,
    prefix_start_index: u32,
    dict_end: *const u8,
    prefix_start: *const u8,
    hbits_long: u32,
    hbits_small: u32,
    mls: u32,
    seq_store: *mut SeqStore_t,
    offset1: &mut u32,
    offset2: &mut u32,
    mut ip: *const u8,
    match_length: usize,
    current: u32,
    iend: *const u8,
    ilimit: *const u8,
) -> (*const u8, *const u8) {
    ip = ip.wrapping_add(match_length);
    let mut anchor = ip;
    if ptr_le(ip, ilimit) {
        let index_to_insert = current.wrapping_add(2);
        let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
        unsafe { table_set(hash_long, hash, index_to_insert) };
        let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
        unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
        let hash = unsafe {
            hash_ptr(
                base.wrapping_add(index_to_insert as usize),
                hbits_small,
                mls,
            )
        };
        unsafe { table_set(hash_small, hash, index_to_insert) };
        let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
        unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };

        while ptr_le(ip, ilimit) {
            let current2 = unsafe { index_from(base, ip) };
            let rep_index2 = current2.wrapping_sub(*offset2);
            let rep_match2 = if rep_index2 < prefix_start_index {
                dict_base.wrapping_add(rep_index2 as usize)
            } else {
                base.wrapping_add(rep_index2 as usize)
            };
            if index_overlap_check(prefix_start_index, rep_index2)
                && *offset2 <= current2.wrapping_sub(dict_start_index)
                && unsafe { read32(rep_match2) == read32(ip) }
            {
                let rep_end2 = if rep_index2 < prefix_start_index {
                    dict_end
                } else {
                    iend
                };
                let repeat_length = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        rep_match2.wrapping_add(4),
                        iend,
                        rep_end2,
                        prefix_start,
                    ) + 4
                };
                std::mem::swap(offset1, offset2);
                unsafe {
                    store_seq(
                        seq_store,
                        0,
                        anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        repeat_length,
                    )
                };
                let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
                unsafe { table_set(hash_small, hash, current2) };
                let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
                unsafe { table_set(hash_long, hash, current2) };
                ip = ip.wrapping_add(repeat_length);
                anchor = ip;
                continue;
            }
            break;
        }
    }
    (ip, anchor)
}

unsafe fn compress_block_double_fast_ext_dict(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    dict_base: *const u8,
    dict_limit: u32,
    low_limit: u32,
    loaded_dict_end: u32,
    hash_log: u32,
    chain_log: u32,
    window_log: u32,
    seq_store: *mut SeqStore_t,
    reps: *mut u32,
    src: *const u8,
    src_size: usize,
    mls: u32,
) -> usize {
    if src_size < HASH_READ_SIZE {
        return src_size;
    }
    let istart = src;
    let iend = istart.wrapping_add(src_size);
    let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
    let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
    let dict_start_index = lowest_match_index(low_limit, loaded_dict_end, end_index, window_log);
    let prefix_start_index = dict_limit.max(dict_start_index);
    if prefix_start_index == dict_start_index {
        return unsafe {
            compress_block_double_fast_no_dict(
                hash_long,
                hash_small,
                base,
                dict_limit,
                loaded_dict_end,
                hash_log,
                chain_log,
                window_log,
                seq_store,
                reps,
                src,
                src_size,
                mls,
            )
        };
    }
    let prefix_start = base.wrapping_add(prefix_start_index as usize);
    let dict_start = dict_base.wrapping_add(dict_start_index as usize);
    let dict_end = dict_base.wrapping_add(prefix_start_index as usize);
    let mut ip = istart;
    let mut anchor = istart;
    let mut offset1 = unsafe { *reps };
    let mut offset2 = unsafe { *reps.add(1) };

    while ptr_lt(ip, ilimit) {
        let h_small = unsafe { hash_ptr(ip, chain_log, mls) };
        let match_index = unsafe { table_get(hash_small, h_small) };
        let match_base = if match_index < prefix_start_index {
            dict_base
        } else {
            base
        };
        let mut matched = match_base.wrapping_add(match_index as usize);
        let h_long = unsafe { hash_ptr(ip, hash_log, 8) };
        let match_long_index = unsafe { table_get(hash_long, h_long) };
        let match_long_base = if match_long_index < prefix_start_index {
            dict_base
        } else {
            base
        };
        let mut match_long = match_long_base.wrapping_add(match_long_index as usize);
        let current = unsafe { index_from(base, ip) };
        let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
        let rep_base = if rep_index < prefix_start_index {
            dict_base
        } else {
            base
        };
        let rep_match = rep_base.wrapping_add(rep_index as usize);
        unsafe {
            table_set(hash_small, h_small, current);
            table_set(hash_long, h_long, current);
        }

        if index_overlap_check(prefix_start_index, rep_index)
            && offset1 <= current.wrapping_add(1).wrapping_sub(dict_start_index)
            && unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
        {
            let rep_end = if rep_index < prefix_start_index {
                dict_end
            } else {
                iend
            };
            let match_length = unsafe {
                count_2segments(
                    ip.wrapping_add(5),
                    rep_match.wrapping_add(4),
                    iend,
                    rep_end,
                    prefix_start,
                ) + 4
            };
            ip = ip.wrapping_add(1);
            unsafe {
                store_seq(
                    seq_store,
                    ip.offset_from(anchor) as usize,
                    anchor,
                    iend,
                    REPCODE1_TO_OFFBASE,
                    match_length,
                )
            };
            let (next_ip, next_anchor) = unsafe {
                post_ext_match(
                    hash_long,
                    hash_small,
                    base,
                    dict_base,
                    dict_start_index,
                    prefix_start_index,
                    dict_end,
                    prefix_start,
                    hash_log,
                    chain_log,
                    mls,
                    seq_store,
                    &mut offset1,
                    &mut offset2,
                    ip,
                    match_length,
                    current,
                    iend,
                    ilimit,
                )
            };
            ip = next_ip;
            anchor = next_anchor;
            continue;
        }

        if match_long_index > dict_start_index && unsafe { read64(match_long) == read64(ip) } {
            let match_end = if match_long_index < prefix_start_index {
                dict_end
            } else {
                iend
            };
            let low_match = if match_long_index < prefix_start_index {
                dict_start
            } else {
                prefix_start
            };
            let mut match_length = unsafe {
                count_2segments(
                    ip.wrapping_add(8),
                    match_long.wrapping_add(8),
                    iend,
                    match_end,
                    prefix_start,
                ) + 8
            };
            let offset = current.wrapping_sub(match_long_index);
            while ptr_gt(ip, anchor)
                && ptr_gt(match_long, low_match)
                && unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) }
            {
                ip = ip.wrapping_sub(1);
                match_long = match_long.wrapping_sub(1);
                match_length += 1;
            }
            offset2 = offset1;
            offset1 = offset;
            unsafe {
                store_seq(
                    seq_store,
                    ip.offset_from(anchor) as usize,
                    anchor,
                    iend,
                    offset.wrapping_add(ZSTD_REP_NUM as u32),
                    match_length,
                )
            };
            let (next_ip, next_anchor) = unsafe {
                post_ext_match(
                    hash_long,
                    hash_small,
                    base,
                    dict_base,
                    dict_start_index,
                    prefix_start_index,
                    dict_end,
                    prefix_start,
                    hash_log,
                    chain_log,
                    mls,
                    seq_store,
                    &mut offset1,
                    &mut offset2,
                    ip,
                    match_length,
                    current,
                    iend,
                    ilimit,
                )
            };
            ip = next_ip;
            anchor = next_anchor;
            continue;
        }

        if match_index > dict_start_index && unsafe { read32(matched) == read32(ip) } {
            let h_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) };
            let match_index3 = unsafe { table_get(hash_long, h_long3) };
            let match_base3 = if match_index3 < prefix_start_index {
                dict_base
            } else {
                base
            };
            let mut match3 = match_base3.wrapping_add(match_index3 as usize);
            unsafe { table_set(hash_long, h_long3, current.wrapping_add(1)) };
            let (match_length, offset) = if match_index3 > dict_start_index
                && unsafe { read64(match3) == read64(ip.wrapping_add(1)) }
            {
                let match_end = if match_index3 < prefix_start_index {
                    dict_end
                } else {
                    iend
                };
                let low_match = if match_index3 < prefix_start_index {
                    dict_start
                } else {
                    prefix_start
                };
                let mut length = unsafe {
                    count_2segments(
                        ip.wrapping_add(9),
                        match3.wrapping_add(8),
                        iend,
                        match_end,
                        prefix_start,
                    ) + 8
                };
                ip = ip.wrapping_add(1);
                let offset = current.wrapping_add(1).wrapping_sub(match_index3);
                while ptr_gt(ip, anchor)
                    && ptr_gt(match3, low_match)
                    && unsafe { *ip.wrapping_sub(1) == *match3.wrapping_sub(1) }
                {
                    ip = ip.wrapping_sub(1);
                    match3 = match3.wrapping_sub(1);
                    length += 1;
                }
                (length, offset)
            } else {
                let match_end = if match_index < prefix_start_index {
                    dict_end
                } else {
                    iend
                };
                let low_match = if match_index < prefix_start_index {
                    dict_start
                } else {
                    prefix_start
                };
                let mut length = unsafe {
                    count_2segments(
                        ip.wrapping_add(4),
                        matched.wrapping_add(4),
                        iend,
                        match_end,
                        prefix_start,
                    ) + 4
                };
                let offset = current.wrapping_sub(match_index);
                while ptr_gt(ip, anchor)
                    && ptr_gt(matched, low_match)
                    && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
                {
                    ip = ip.wrapping_sub(1);
                    matched = matched.wrapping_sub(1);
                    length += 1;
                }
                (length, offset)
            };
            offset2 = offset1;
            offset1 = offset;
            unsafe {
                store_seq(
                    seq_store,
                    ip.offset_from(anchor) as usize,
                    anchor,
                    iend,
                    offset.wrapping_add(ZSTD_REP_NUM as u32),
                    match_length,
                )
            };
            let (next_ip, next_anchor) = unsafe {
                post_ext_match(
                    hash_long,
                    hash_small,
                    base,
                    dict_base,
                    dict_start_index,
                    prefix_start_index,
                    dict_end,
                    prefix_start,
                    hash_log,
                    chain_log,
                    mls,
                    seq_store,
                    &mut offset1,
                    &mut offset2,
                    ip,
                    match_length,
                    current,
                    iend,
                    ilimit,
                )
            };
            ip = next_ip;
            anchor = next_anchor;
            continue;
        }

        ip = ip.wrapping_add((unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1);
    }

    unsafe {
        *reps = offset1;
        *reps.add(1) = offset2;
    }
    unsafe { iend.offset_from(anchor) as usize }
}

/// Rust implementation called by the C ABI wrapper for external dictionaries.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_extDict(
    hash_long: *mut u32,
    hash_small: *mut u32,
    base: *const u8,
    dict_base: *const u8,
    dict_limit: u32,
    low_limit: u32,
    loaded_dict_end: u32,
    hash_log: u32,
    chain_log: u32,
    min_match: u32,
    window_log: u32,
    seq_store: *mut c_void,
    reps: *mut u32,
    src: *const c_void,
    src_size: usize,
) -> usize {
    unsafe {
        compress_block_double_fast_ext_dict(
            hash_long,
            hash_small,
            base,
            dict_base,
            dict_limit,
            low_limit,
            loaded_dict_end,
            hash_log,
            chain_log,
            window_log,
            seq_store.cast::<SeqStore_t>(),
            reps,
            src.cast::<u8>(),
            src_size,
            fast_mls(min_match),
        )
    }
}

#[cfg(test)]
mod tests {
    use super::{attached_rep_index, index_overlap_check};

    #[test]
    fn index_overlap_check_preserves_dictionary_and_prefix_boundaries() {
        assert!(index_overlap_check(16, 12));
        assert!(!index_overlap_check(16, 13));
        assert!(index_overlap_check(16, 16));
    }

    #[test]
    fn attached_rep_index_rejects_wrapped_post_match_coordinate() {
        // This is the failing shape: 0x1fb - 0x770 wrapped to 0xfffffa8b.
        assert_eq!(attached_rep_index(0, 0, 0, 0x1fb, 0, 0x770), None);
    }

    #[test]
    fn attached_rep_index_checks_dictionary_coordinate_boundaries() {
        let prefix_lowest_index = 100;
        let dict_start_index = 10;
        let dict_index_delta = 20;
        let current = 120;

        // The translated dictionary interval is [30, 100), with four bytes
        // available at the upper boundary only through index 96.
        assert_eq!(
            attached_rep_index(
                prefix_lowest_index,
                dict_start_index,
                dict_index_delta,
                current,
                0,
                90,
            ),
            Some(30)
        );
        assert_eq!(
            attached_rep_index(
                prefix_lowest_index,
                dict_start_index,
                dict_index_delta,
                current,
                0,
                91,
            ),
            None
        );
        assert_eq!(
            attached_rep_index(
                prefix_lowest_index,
                dict_start_index,
                dict_index_delta,
                current,
                0,
                24,
            ),
            Some(96)
        );
        assert_eq!(
            attached_rep_index(
                prefix_lowest_index,
                dict_start_index,
                dict_index_delta,
                current,
                0,
                23,
            ),
            None
        );
    }

    #[test]
    fn attached_rep_index_accepts_prefix_boundary_but_not_zero_offset() {
        assert_eq!(attached_rep_index(100, 10, 20, 120, 0, 20), Some(100));
        assert_eq!(attached_rep_index(100, 10, 20, 120, 1, 21), Some(100));
        assert_eq!(attached_rep_index(100, 10, 20, 120, 1, 0), None);
    }
}