zstd-rs/rust/src/zstd_fast.rs

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]

//! Fast block match finder.
//!
//! The C translation unit keeps the public/internal ABI entry points and
//! extracts the fields it needs from `ZSTD_MatchState_t`.  This module owns
//! the hash-table search and sequence-generation algorithms themselves.  That
//! boundary keeps the large, evolving match-state opaque while retaining the
//! hot matching loops in Rust.

use crate::mem::{
    MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST,
};
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::c_int;
use std::ptr;

const ZSTD_REP_NUM: usize = 3;
const MINMATCH: usize = 3;
const HASH_READ_SIZE: usize = 8;
const SHORT_CACHE_TAG_BITS: u32 = 8;
const SHORT_CACHE_TAG_MASK: u32 = (1 << SHORT_CACHE_TAG_BITS) - 1;
const K_SEARCH_STRENGTH: usize = 8;
const REPCODE1_TO_OFFBASE: u32 = 1;

#[repr(C)]
#[derive(Clone, Copy)]
struct SeqDef {
    offBase: u32,
    litLength: u16,
    mlBase: u16,
}

/// The only leaf layout written by the matcher.  The C shim verifies this
/// shape at compile time; `ZSTD_MatchState_t` itself never crosses the FFI.
#[repr(C)]
struct SeqStore_t {
    sequencesStart: *mut SeqDef,
    sequences: *mut SeqDef,
    litStart: *mut u8,
    lit: *mut u8,
    llCode: *mut u8,
    mlCode: *mut u8,
    ofCode: *mut u8,
    maxNbSeq: usize,
    maxNbLit: usize,
    longLengthType: c_int,
    longLengthPos: u32,
}

#[inline]
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
    (left as usize) < (right as usize)
}

#[inline]
fn ptr_le(left: *const u8, right: *const u8) -> bool {
    (left as usize) <= (right as usize)
}

#[inline]
fn ptr_ge(left: *const u8, right: *const u8) -> bool {
    (left as usize) >= (right as usize)
}

#[inline]
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
    (left as usize) > (right as usize)
}

#[inline]
unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 {
    unsafe { ptr.offset_from(base) as u32 }
}

#[inline]
unsafe fn read32(ptr: *const u8) -> u32 {
    unsafe { MEM_read32(ptr.cast::<c_void>()) }
}

#[inline]
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
    unsafe { *table.add(index) }
}

#[inline]
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
    unsafe { *table.add(index) = value };
}

#[inline]
fn hash_shift32(value: u32, hbits: u32) -> usize {
    if hbits == 0 {
        0
    } else {
        (value >> (32 - hbits)) as usize
    }
}

#[inline]
fn hash_shift64(value: u64, hbits: u32) -> usize {
    if hbits == 0 {
        0
    } else {
        (value >> (64 - hbits)) as usize
    }
}

#[inline]
unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize {
    match mls {
        5 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits)
        }
        6 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(16).wrapping_mul(227_718_039_650_203),
                hbits,
            )
        }
        7 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(
                value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627),
                hbits,
            )
        }
        8 => {
            let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
            hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits)
        }
        _ => {
            let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
            hash_shift32(value.wrapping_mul(2_654_435_761), hbits)
        }
    }
}

#[inline]
fn common_bytes(word: usize) -> usize {
    let zeros = if MEM_isLittleEndian() {
        word.trailing_zeros()
    } else {
        word.leading_zeros()
    };
    (zeros / 8) as usize
}

/// Equivalent to C's `ZSTD_count()`, including its word-at-a-time fast path.
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
    let input_start = input;
    let word_size = size_of::<usize>();

    while unsafe { input_limit.offset_from(input) as usize } >= word_size {
        let diff =
            unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
        if diff != 0 {
            return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff);
        }
        input = input.wrapping_add(word_size);
        matched = matched.wrapping_add(word_size);
    }

    if MEM_64bits()
        && unsafe { input_limit.offset_from(input) as usize } >= 4
        && unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
    {
        input = input.wrapping_add(4);
        matched = matched.wrapping_add(4);
    }
    if unsafe { input_limit.offset_from(input) as usize } >= 2
        && unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
    {
        input = input.wrapping_add(2);
        matched = matched.wrapping_add(2);
    }
    if ptr_lt(input, input_limit) && unsafe { *matched == *input } {
        input = input.wrapping_add(1);
    }
    unsafe { input.offset_from(input_start) as usize }
}

unsafe fn count_2segments(
    input: *const u8,
    matched: *const u8,
    input_end: *const u8,
    match_end: *const u8,
    input_start: *const u8,
) -> usize {
    let match_remaining = unsafe { match_end.offset_from(matched) as usize };
    let input_remaining = unsafe { input_end.offset_from(input) as usize };
    let first_end = input.wrapping_add(match_remaining.min(input_remaining));
    let first_count = unsafe { count(input, matched, first_end) };
    if matched.wrapping_add(first_count) != match_end {
        return first_count;
    }
    first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
}

#[inline]
fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
    let max_distance = 1u32.wrapping_shl(window_log);
    let within_window = if curr.wrapping_sub(dict_limit) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        dict_limit
    };
    if loaded_dict_end != 0 {
        dict_limit
    } else {
        within_window
    }
}

#[inline]
fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
    let max_distance = 1u32.wrapping_shl(window_log);
    let within_window = if curr.wrapping_sub(low_limit) > max_distance {
        curr.wrapping_sub(max_distance)
    } else {
        low_limit
    };
    if loaded_dict_end != 0 {
        low_limit
    } else {
        within_window
    }
}

#[inline]
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
    prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
}

#[inline]
fn write_tagged_index(table: *mut u32, hash_and_tag: usize, index: u32) {
    let hash = hash_and_tag >> SHORT_CACHE_TAG_BITS;
    let tag = (hash_and_tag as u32) & SHORT_CACHE_TAG_MASK;
    unsafe { table_set(table, hash, (index << SHORT_CACHE_TAG_BITS) | tag) };
}

#[inline]
fn packed_tags_match(first: u32, second: usize) -> bool {
    (first & SHORT_CACHE_TAG_MASK) == ((second as u32) & SHORT_CACHE_TAG_MASK)
}

/// Stores a sequence using the C `SeqStore_t` ABI.  Copying exactly the
/// literal range is equivalent to C's over-copying wildcopy path for all
/// observable sequence-store bytes and avoids speculative reads in Rust.
unsafe fn store_seq(
    seq_store: *mut SeqStore_t,
    lit_length: usize,
    literals: *const u8,
    _lit_limit: *const u8,
    off_base: u32,
    match_length: usize,
) {
    let seq_store = unsafe { &mut *seq_store };
    let sequence = seq_store.sequences;
    debug_assert!(
        unsafe { sequence.offset_from(seq_store.sequencesStart) as usize } < seq_store.maxNbSeq
    );
    debug_assert!(lit_length <= seq_store.maxNbLit);
    debug_assert!(match_length >= MINMATCH);

    if lit_length != 0 {
        unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) };
    }
    seq_store.lit = seq_store.lit.wrapping_add(lit_length);

    let sequence_index = unsafe { sequence.offset_from(seq_store.sequencesStart) as u32 };
    if lit_length > u16::MAX as usize {
        debug_assert_eq!(seq_store.longLengthType, 0);
        seq_store.longLengthType = 1;
        seq_store.longLengthPos = sequence_index;
    }
    unsafe { (*sequence).litLength = lit_length as u16 };
    unsafe { (*sequence).offBase = off_base };

    let match_base = match_length - MINMATCH;
    if match_base > u16::MAX as usize {
        debug_assert_eq!(seq_store.longLengthType, 0);
        seq_store.longLengthType = 2;
        seq_store.longLengthPos = sequence_index;
    }
    unsafe { (*sequence).mlBase = match_base as u16 };
    seq_store.sequences = sequence.wrapping_add(1);
}

/// Reuse the sequence-store leaf from another Rust compression module without
/// exposing this module's C-layout projection in its public API.
pub(crate) unsafe fn store_seq_opaque(
    seq_store: *mut c_void,
    lit_length: usize,
    literals: *const c_void,
    lit_limit: *const c_void,
    off_base: u32,
    match_length: usize,
) {
    unsafe {
        store_seq(
            seq_store.cast::<SeqStore_t>(),
            lit_length,
            literals.cast::<u8>(),
            lit_limit.cast::<u8>(),
            off_base,
            match_length,
        )
    }
}

#[inline]
unsafe fn match4_found(
    current: *const u8,
    match_address: *const u8,
    match_index: u32,
    low_limit: u32,
) -> bool {
    match_index >= low_limit && unsafe { read32(current) == read32(match_address) }
}

unsafe fn finish_no_dict_match(
    hash_table: *mut u32,
    base: *const u8,
    hash_log: u32,
    mls: u32,
    seq_store: *mut SeqStore_t,
    rep_offset1: &mut u32,
    rep_offset2: &mut u32,
    mut ip0: *const u8,
    match0: *const u8,
    anchor: *const u8,
    iend: *const u8,
    ilimit: *const u8,
    current0: u32,
    offcode: u32,
    mut match_length: usize,
) -> (*const u8, *const u8) {
    match_length += unsafe {
        count(
            ip0.wrapping_add(match_length),
            match0.wrapping_add(match_length),
            iend,
        )
    };
    unsafe {
        store_seq(
            seq_store,
            ip0.offset_from(anchor) as usize,
            anchor,
            iend,
            offcode,
            match_length,
        )
    };
    ip0 = ip0.wrapping_add(match_length);
    let mut new_anchor = ip0;

    if ptr_le(ip0, ilimit) {
        let table_hash =
            unsafe { hash_ptr(base.wrapping_add(current0 as usize + 2), hash_log, mls) };
        unsafe { table_set(hash_table, table_hash, current0.wrapping_add(2)) };
        let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) };
        unsafe {
            table_set(
                hash_table,
                table_hash,
                index_from(base, ip0.wrapping_sub(2)),
            )
        };

        if *rep_offset2 > 0 {
            while ptr_le(ip0, ilimit)
                && unsafe { read32(ip0) == read32(ip0.wrapping_sub(*rep_offset2 as usize)) }
            {
                let repeat_length = unsafe {
                    count(
                        ip0.wrapping_add(4),
                        ip0.wrapping_add(4).wrapping_sub(*rep_offset2 as usize),
                        iend,
                    ) + 4
                };
                std::mem::swap(rep_offset1, rep_offset2);
                let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) };
                unsafe { table_set(hash_table, table_hash, index_from(base, ip0)) };
                unsafe {
                    store_seq(
                        seq_store,
                        0,
                        new_anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        repeat_length,
                    )
                };
                ip0 = ip0.wrapping_add(repeat_length);
                new_anchor = ip0;
            }
        }
    }

    (ip0, new_anchor)
}

unsafe fn compress_block_fast_no_dict(
    hash_table: *mut u32,
    base: *const u8,
    dict_limit: u32,
    loaded_dict_end: u32,
    hash_log: u32,
    target_length: u32,
    window_log: u32,
    seq_store: *mut SeqStore_t,
    reps: *mut u32,
    src: *const u8,
    src_size: usize,
    mls: u32,
    _use_cmov: bool,
) -> usize {
    if src_size < HASH_READ_SIZE {
        return src_size;
    }
    let istart = src;
    let iend = istart.wrapping_add(src_size);
    let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
    let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
    let prefix_start_index =
        lowest_prefix_index(dict_limit, loaded_dict_end, end_index, window_log);
    let prefix_start = base.wrapping_add(prefix_start_index as usize);
    let step_size = target_length as usize + usize::from(target_length == 0) + 1;
    let mut ip0 = istart;
    let mut anchor = istart;
    let mut rep_offset1 = unsafe { *reps };
    let mut rep_offset2 = unsafe { *reps.add(1) };
    let mut offset_saved1 = 0u32;
    let mut offset_saved2 = 0u32;

    if ip0 == prefix_start {
        ip0 = ip0.wrapping_add(1);
    }
    let current = unsafe { index_from(base, ip0) };
    let window_low = lowest_prefix_index(dict_limit, loaded_dict_end, current, window_log);
    let max_rep = current.wrapping_sub(window_low);
    if rep_offset2 > max_rep {
        offset_saved2 = rep_offset2;
        rep_offset2 = 0;
    }
    if rep_offset1 > max_rep {
        offset_saved1 = rep_offset1;
        rep_offset1 = 0;
    }

    'start: loop {
        let mut step = step_size;
        let mut next_step = ip0.wrapping_add(1 << (K_SEARCH_STRENGTH - 1));
        let mut ip1 = ip0.wrapping_add(1);
        let mut ip2 = ip0.wrapping_add(step);
        let mut ip3 = ip2.wrapping_add(1);
        if ptr_ge(ip3, ilimit) {
            break;
        }

        let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) };
        let mut hash1 = unsafe { hash_ptr(ip1, hash_log, mls) };
        let mut match_index = unsafe { table_get(hash_table, hash0) };

        loop {
            let rval = if rep_offset1 > 0 {
                unsafe { read32(ip2.wrapping_sub(rep_offset1 as usize)) }
            } else {
                unsafe { read32(ip2) }
            };
            let mut current0 = unsafe { index_from(base, ip0) };
            unsafe { table_set(hash_table, hash0, current0) };

            if rep_offset1 > 0 && unsafe { read32(ip2) == rval } {
                ip0 = ip2;
                let mut match0 = ip0.wrapping_sub(rep_offset1 as usize);
                let match_length =
                    usize::from(unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) });
                ip0 = ip0.wrapping_sub(match_length);
                match0 = match0.wrapping_sub(match_length);
                unsafe { table_set(hash_table, hash1, index_from(base, ip1)) };
                let (new_ip0, new_anchor) = unsafe {
                    finish_no_dict_match(
                        hash_table,
                        base,
                        hash_log,
                        mls,
                        seq_store,
                        &mut rep_offset1,
                        &mut rep_offset2,
                        ip0,
                        match0,
                        anchor,
                        iend,
                        ilimit,
                        current0,
                        REPCODE1_TO_OFFBASE,
                        match_length + 4,
                    )
                };
                ip0 = new_ip0;
                anchor = new_anchor;
                continue 'start;
            }

            if unsafe {
                match4_found(
                    ip0,
                    base.wrapping_add(match_index as usize),
                    match_index,
                    prefix_start_index,
                )
            } {
                unsafe { table_set(hash_table, hash1, index_from(base, ip1)) };
                let mut match0 = base.wrapping_add(match_index as usize);
                rep_offset2 = rep_offset1;
                rep_offset1 = unsafe { index_from(match0, ip0) };
                let mut match_length = 4usize;
                while ptr_gt(ip0, anchor)
                    && ptr_gt(match0, prefix_start)
                    && unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }
                {
                    ip0 = ip0.wrapping_sub(1);
                    match0 = match0.wrapping_sub(1);
                    match_length += 1;
                }
                let offcode = rep_offset1.wrapping_add(ZSTD_REP_NUM as u32);
                let (new_ip0, new_anchor) = unsafe {
                    finish_no_dict_match(
                        hash_table,
                        base,
                        hash_log,
                        mls,
                        seq_store,
                        &mut rep_offset1,
                        &mut rep_offset2,
                        ip0,
                        match0,
                        anchor,
                        iend,
                        ilimit,
                        current0,
                        offcode,
                        match_length,
                    )
                };
                ip0 = new_ip0;
                anchor = new_anchor;
                continue 'start;
            }

            match_index = unsafe { table_get(hash_table, hash1) };
            hash0 = hash1;
            hash1 = unsafe { hash_ptr(ip2, hash_log, mls) };
            ip0 = ip1;
            ip1 = ip2;
            ip2 = ip3;
            current0 = unsafe { index_from(base, ip0) };
            unsafe { table_set(hash_table, hash0, current0) };

            if unsafe {
                match4_found(
                    ip0,
                    base.wrapping_add(match_index as usize),
                    match_index,
                    prefix_start_index,
                )
            } {
                if step <= 4 {
                    unsafe { table_set(hash_table, hash1, index_from(base, ip1)) };
                }
                let mut match0 = base.wrapping_add(match_index as usize);
                rep_offset2 = rep_offset1;
                rep_offset1 = unsafe { index_from(match0, ip0) };
                let mut match_length = 4usize;
                while ptr_gt(ip0, anchor)
                    && ptr_gt(match0, prefix_start)
                    && unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }
                {
                    ip0 = ip0.wrapping_sub(1);
                    match0 = match0.wrapping_sub(1);
                    match_length += 1;
                }
                let offcode = rep_offset1.wrapping_add(ZSTD_REP_NUM as u32);
                let (new_ip0, new_anchor) = unsafe {
                    finish_no_dict_match(
                        hash_table,
                        base,
                        hash_log,
                        mls,
                        seq_store,
                        &mut rep_offset1,
                        &mut rep_offset2,
                        ip0,
                        match0,
                        anchor,
                        iend,
                        ilimit,
                        current0,
                        offcode,
                        match_length,
                    )
                };
                ip0 = new_ip0;
                anchor = new_anchor;
                continue 'start;
            }

            match_index = unsafe { table_get(hash_table, hash1) };
            hash0 = hash1;
            hash1 = unsafe { hash_ptr(ip2, hash_log, mls) };
            ip0 = ip1;
            ip1 = ip2;
            ip2 = ip0.wrapping_add(step);
            ip3 = ip1.wrapping_add(step);
            if ptr_ge(ip2, next_step) {
                step += 1;
                next_step = next_step.wrapping_add(1 << (K_SEARCH_STRENGTH - 1));
            }
            if ptr_ge(ip3, ilimit) {
                break 'start;
            }
        }
    }

    offset_saved2 = if offset_saved1 != 0 && rep_offset1 != 0 {
        offset_saved1
    } else {
        offset_saved2
    };
    unsafe {
        *reps = if rep_offset1 != 0 {
            rep_offset1
        } else {
            offset_saved1
        };
        *reps.add(1) = if rep_offset2 != 0 {
            rep_offset2
        } else {
            offset_saved2
        };
    }
    unsafe { iend.offset_from(anchor) as usize }
}

unsafe fn fill_hash_table(
    hash_table: *mut u32,
    base: *const u8,
    next_to_update: u32,
    end: *const u8,
    hash_log: u32,
    min_match: u32,
    full_table_load: bool,
    tagged_indices: bool,
) {
    if unsafe { end.offset_from(base) } < HASH_READ_SIZE as isize {
        return;
    }
    let hbits = hash_log
        + if tagged_indices {
            SHORT_CACHE_TAG_BITS
        } else {
            0
        };
    let mut input = base.wrapping_add(next_to_update as usize);
    let input_end = end.wrapping_sub(HASH_READ_SIZE);

    while ptr_lt(input.wrapping_add(3), input_end.wrapping_add(2)) {
        let current = unsafe { index_from(base, input) };
        let hash_and_tag = unsafe { hash_ptr(input, hbits, min_match) };
        if tagged_indices {
            write_tagged_index(hash_table, hash_and_tag, current);
        } else {
            unsafe { table_set(hash_table, hash_and_tag, current) };
        }

        if full_table_load {
            for position in 1..3usize {
                let hash_and_tag =
                    unsafe { hash_ptr(input.wrapping_add(position), hbits, min_match) };
                let table_index = if tagged_indices {
                    hash_and_tag >> SHORT_CACHE_TAG_BITS
                } else {
                    hash_and_tag
                };
                if unsafe { table_get(hash_table, table_index) } == 0 {
                    if tagged_indices {
                        write_tagged_index(
                            hash_table,
                            hash_and_tag,
                            current.wrapping_add(position as u32),
                        );
                    } else {
                        unsafe {
                            table_set(
                                hash_table,
                                table_index,
                                current.wrapping_add(position as u32),
                            )
                        };
                    }
                }
            }
        }
        input = input.wrapping_add(3);
    }
}

#[inline]
fn fast_mls(min_match: u32) -> u32 {
    match min_match {
        5..=7 => min_match,
        _ => 4,
    }
}

/// Rust implementation called by the C ABI wrapper for `ZSTD_fillHashTable`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_fillHashTable(
    hash_table: *mut u32,
    base: *const u8,
    next_to_update: u32,
    end: *const c_void,
    hash_log: u32,
    min_match: u32,
    full_table_load: c_int,
    for_cdict: c_int,
) {
    unsafe {
        fill_hash_table(
            hash_table,
            base,
            next_to_update,
            end.cast::<u8>(),
            hash_log,
            min_match,
            full_table_load != 0,
            for_cdict != 0,
        )
    };
}

/// Rust implementation called by the C ABI wrapper for `ZSTD_compressBlock_fast`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast(
    hash_table: *mut u32,
    base: *const u8,
    dict_limit: u32,
    loaded_dict_end: u32,
    hash_log: u32,
    min_match: u32,
    target_length: u32,
    window_log: u32,
    seq_store: *mut c_void,
    reps: *mut u32,
    src: *const c_void,
    src_size: usize,
) -> usize {
    unsafe {
        compress_block_fast_no_dict(
            hash_table,
            base,
            dict_limit,
            loaded_dict_end,
            hash_log,
            target_length,
            window_log,
            seq_store.cast::<SeqStore_t>(),
            reps,
            src.cast::<u8>(),
            src_size,
            fast_mls(min_match),
            window_log < 19,
        )
    }
}

unsafe fn compress_block_fast_dict_match_state(
    hash_table: *mut u32,
    base: *const u8,
    prefix_start_index: u32,
    hash_log: u32,
    target_length: u32,
    seq_store: *mut SeqStore_t,
    reps: *mut u32,
    src: *const u8,
    src_size: usize,
    mls: u32,
    dict_hash_table: *const u32,
    dict_base: *const u8,
    dict_start_index: u32,
    dict_end: *const u8,
    dict_hash_log: u32,
    _prefetch_cdict_tables: bool,
) -> usize {
    if src_size < HASH_READ_SIZE {
        return src_size;
    }
    let istart = src;
    let iend = istart.wrapping_add(src_size);
    let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
    let prefix_start = base.wrapping_add(prefix_start_index as usize);
    let dict_start = dict_base.wrapping_add(dict_start_index as usize);
    let dict_index_delta =
        prefix_start_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) });
    let dict_and_prefix_length = unsafe {
        istart.offset_from(prefix_start) as u32 + dict_end.offset_from(dict_start) as u32
    };
    let dict_hbits = dict_hash_log + SHORT_CACHE_TAG_BITS;
    let step_size = target_length + u32::from(target_length == 0);
    let mut ip0 = istart;
    let mut ip1 = ip0.wrapping_add(step_size as usize);
    let mut anchor = istart;
    let mut offset1 = unsafe { *reps };
    let mut offset2 = unsafe { *reps.add(1) };

    if dict_and_prefix_length == 0 {
        ip0 = ip0.wrapping_add(1);
    }

    'outer: while ptr_le(ip1, ilimit) {
        let mut match_length: usize;
        let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) };
        let dict_hash_and_tag0 = unsafe { hash_ptr(ip0, dict_hbits, mls) };
        let mut dict_match_index_and_tag =
            unsafe { table_get(dict_hash_table, dict_hash_and_tag0 >> SHORT_CACHE_TAG_BITS) };
        let mut dict_tags_match = packed_tags_match(dict_match_index_and_tag, dict_hash_and_tag0);
        let mut match_index = unsafe { table_get(hash_table, hash0) };
        let mut current = unsafe { index_from(base, ip0) };
        let mut step = step_size as usize;
        let mut next_step = ip0.wrapping_add(1 << K_SEARCH_STRENGTH);

        loop {
            let match_ptr = base.wrapping_add(match_index as usize);
            let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
            let rep_match = if rep_index < prefix_start_index {
                dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
            } else {
                base.wrapping_add(rep_index as usize)
            };
            let hash1 = unsafe { hash_ptr(ip1, hash_log, mls) };
            let dict_hash_and_tag1 = unsafe { hash_ptr(ip1, dict_hbits, mls) };
            unsafe { table_set(hash_table, hash0, current) };

            if index_overlap_check(prefix_start_index, rep_index)
                && unsafe { read32(rep_match) == read32(ip0.wrapping_add(1)) }
            {
                let rep_match_end = if rep_index < prefix_start_index {
                    dict_end
                } else {
                    iend
                };
                match_length = unsafe {
                    count_2segments(
                        ip0.wrapping_add(5),
                        rep_match.wrapping_add(4),
                        iend,
                        rep_match_end,
                        prefix_start,
                    ) + 4
                };
                ip0 = ip0.wrapping_add(1);
                unsafe {
                    store_seq(
                        seq_store,
                        ip0.offset_from(anchor) as usize,
                        anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        match_length,
                    )
                };
                break;
            }

            if dict_tags_match {
                let dict_match_index = dict_match_index_and_tag >> SHORT_CACHE_TAG_BITS;
                let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
                if dict_match_index > dict_start_index
                    && unsafe { read32(dict_match) == read32(ip0) }
                    && match_index <= prefix_start_index
                {
                    let offset = current
                        .wrapping_sub(dict_match_index)
                        .wrapping_sub(dict_index_delta);
                    match_length = unsafe {
                        count_2segments(
                            ip0.wrapping_add(4),
                            dict_match.wrapping_add(4),
                            iend,
                            dict_end,
                            prefix_start,
                        ) + 4
                    };
                    while ptr_gt(ip0, anchor)
                        && ptr_gt(dict_match, dict_start)
                        && unsafe { *ip0.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
                    {
                        ip0 = ip0.wrapping_sub(1);
                        dict_match = dict_match.wrapping_sub(1);
                        match_length += 1;
                    }
                    offset2 = offset1;
                    offset1 = offset;
                    unsafe {
                        store_seq(
                            seq_store,
                            ip0.offset_from(anchor) as usize,
                            anchor,
                            iend,
                            offset.wrapping_add(ZSTD_REP_NUM as u32),
                            match_length,
                        )
                    };
                    break;
                }
            }

            if unsafe { match4_found(ip0, match_ptr, match_index, prefix_start_index) } {
                let offset = unsafe { index_from(match_ptr, ip0) };
                match_length =
                    unsafe { count(ip0.wrapping_add(4), match_ptr.wrapping_add(4), iend) + 4 };
                let mut matched = match_ptr;
                while ptr_gt(ip0, anchor)
                    && ptr_gt(matched, prefix_start)
                    && unsafe { *ip0.wrapping_sub(1) == *matched.wrapping_sub(1) }
                {
                    ip0 = ip0.wrapping_sub(1);
                    matched = matched.wrapping_sub(1);
                    match_length += 1;
                }
                offset2 = offset1;
                offset1 = offset;
                unsafe {
                    store_seq(
                        seq_store,
                        ip0.offset_from(anchor) as usize,
                        anchor,
                        iend,
                        offset.wrapping_add(ZSTD_REP_NUM as u32),
                        match_length,
                    )
                };
                break;
            }

            dict_match_index_and_tag =
                unsafe { table_get(dict_hash_table, dict_hash_and_tag1 >> SHORT_CACHE_TAG_BITS) };
            dict_tags_match = packed_tags_match(dict_match_index_and_tag, dict_hash_and_tag1);
            match_index = unsafe { table_get(hash_table, hash1) };
            if ptr_ge(ip1, next_step) {
                step += 1;
                next_step = next_step.wrapping_add(1 << K_SEARCH_STRENGTH);
            }
            ip0 = ip1;
            ip1 = ip1.wrapping_add(step);
            if ptr_gt(ip1, ilimit) {
                break 'outer;
            }
            current = unsafe { index_from(base, ip0) };
            hash0 = hash1;
        }

        ip0 = ip0.wrapping_add(match_length);
        anchor = ip0;
        if ptr_le(ip0, ilimit) {
            let table_hash =
                unsafe { hash_ptr(base.wrapping_add(current as usize + 2), hash_log, mls) };
            unsafe { table_set(hash_table, table_hash, current.wrapping_add(2)) };
            let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) };
            unsafe {
                table_set(
                    hash_table,
                    table_hash,
                    index_from(base, ip0.wrapping_sub(2)),
                )
            };

            while ptr_le(ip0, ilimit) {
                let current2 = unsafe { index_from(base, ip0) };
                let rep_index2 = current2.wrapping_sub(offset2);
                let rep_match2 = if rep_index2 < prefix_start_index {
                    dict_base
                        .wrapping_sub(dict_index_delta as usize)
                        .wrapping_add(rep_index2 as usize)
                } else {
                    base.wrapping_add(rep_index2 as usize)
                };
                if index_overlap_check(prefix_start_index, rep_index2)
                    && unsafe { read32(rep_match2) == read32(ip0) }
                {
                    let rep_end2 = if rep_index2 < prefix_start_index {
                        dict_end
                    } else {
                        iend
                    };
                    let repeat_length = unsafe {
                        count_2segments(
                            ip0.wrapping_add(4),
                            rep_match2.wrapping_add(4),
                            iend,
                            rep_end2,
                            prefix_start,
                        ) + 4
                    };
                    std::mem::swap(&mut offset1, &mut offset2);
                    unsafe {
                        store_seq(
                            seq_store,
                            0,
                            anchor,
                            iend,
                            REPCODE1_TO_OFFBASE,
                            repeat_length,
                        )
                    };
                    let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) };
                    unsafe { table_set(hash_table, table_hash, current2) };
                    ip0 = ip0.wrapping_add(repeat_length);
                    anchor = ip0;
                    continue;
                }
                break;
            }
        }
        ip1 = ip0.wrapping_add(step_size as usize);
    }

    unsafe {
        *reps = offset1;
        *reps.add(1) = offset2;
    }
    unsafe { iend.offset_from(anchor) as usize }
}

/// Rust implementation called by the C ABI wrapper for attached dictionaries.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast_dictMatchState(
    hash_table: *mut u32,
    base: *const u8,
    prefix_start_index: u32,
    hash_log: u32,
    min_match: u32,
    target_length: u32,
    seq_store: *mut c_void,
    reps: *mut u32,
    src: *const c_void,
    src_size: usize,
    dict_hash_table: *const u32,
    dict_base: *const u8,
    dict_start_index: u32,
    dict_end: *const u8,
    dict_hash_log: u32,
    prefetch_cdict_tables: c_int,
) -> usize {
    unsafe {
        compress_block_fast_dict_match_state(
            hash_table,
            base,
            prefix_start_index,
            hash_log,
            target_length,
            seq_store.cast::<SeqStore_t>(),
            reps,
            src.cast::<u8>(),
            src_size,
            fast_mls(min_match),
            dict_hash_table,
            dict_base,
            dict_start_index,
            dict_end,
            dict_hash_log,
            prefetch_cdict_tables != 0,
        )
    }
}

unsafe fn finish_ext_dict_match(
    hash_table: *mut u32,
    base: *const u8,
    dict_base: *const u8,
    prefix_start_index: u32,
    dict_end: *const u8,
    prefix_start: *const u8,
    hash_log: u32,
    mls: u32,
    seq_store: *mut SeqStore_t,
    offset1: &mut u32,
    offset2: &mut u32,
    mut ip0: *const u8,
    match0: *const u8,
    match_end: *const u8,
    anchor: *const u8,
    iend: *const u8,
    ilimit: *const u8,
    current0: u32,
    hash1: usize,
    ip1: *const u8,
    offcode: u32,
    mut match_length: usize,
) -> (*const u8, *const u8) {
    match_length += unsafe {
        count_2segments(
            ip0.wrapping_add(match_length),
            match0.wrapping_add(match_length),
            iend,
            match_end,
            prefix_start,
        )
    };
    unsafe {
        store_seq(
            seq_store,
            ip0.offset_from(anchor) as usize,
            anchor,
            iend,
            offcode,
            match_length,
        )
    };
    ip0 = ip0.wrapping_add(match_length);
    let mut new_anchor = ip0;

    if ptr_lt(ip1, ip0) {
        let table_hash = hash1;
        unsafe { table_set(hash_table, table_hash, index_from(base, ip1)) };
    }
    if ptr_le(ip0, ilimit) {
        let table_hash =
            unsafe { hash_ptr(base.wrapping_add(current0 as usize + 2), hash_log, mls) };
        unsafe { table_set(hash_table, table_hash, current0.wrapping_add(2)) };
        let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) };
        unsafe {
            table_set(
                hash_table,
                table_hash,
                index_from(base, ip0.wrapping_sub(2)),
            )
        };

        while ptr_le(ip0, ilimit) {
            let rep_index2 = unsafe { index_from(base, ip0) }.wrapping_sub(*offset2);
            let rep_match2 = if rep_index2 < prefix_start_index {
                dict_base.wrapping_add(rep_index2 as usize)
            } else {
                base.wrapping_add(rep_index2 as usize)
            };
            if *offset2 > 0
                && index_overlap_check(prefix_start_index, rep_index2)
                && unsafe { read32(rep_match2) == read32(ip0) }
            {
                let rep_end2 = if rep_index2 < prefix_start_index {
                    dict_end
                } else {
                    iend
                };
                let repeat_length = unsafe {
                    count_2segments(
                        ip0.wrapping_add(4),
                        rep_match2.wrapping_add(4),
                        iend,
                        rep_end2,
                        prefix_start,
                    ) + 4
                };
                std::mem::swap(offset1, offset2);
                unsafe {
                    store_seq(
                        seq_store,
                        0,
                        new_anchor,
                        iend,
                        REPCODE1_TO_OFFBASE,
                        repeat_length,
                    )
                };
                let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) };
                unsafe { table_set(hash_table, table_hash, index_from(base, ip0)) };
                ip0 = ip0.wrapping_add(repeat_length);
                new_anchor = ip0;
                continue;
            }
            break;
        }
    }
    (ip0, new_anchor)
}

unsafe fn compress_block_fast_ext_dict(
    hash_table: *mut u32,
    base: *const u8,
    dict_base: *const u8,
    dict_limit: u32,
    low_limit: u32,
    loaded_dict_end: u32,
    hash_log: u32,
    target_length: u32,
    window_log: u32,
    seq_store: *mut SeqStore_t,
    reps: *mut u32,
    src: *const u8,
    src_size: usize,
    mls: u32,
) -> usize {
    if src_size < HASH_READ_SIZE {
        return src_size;
    }
    let istart = src;
    let iend = istart.wrapping_add(src_size);
    let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
    let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
    let dict_start_index = lowest_match_index(low_limit, loaded_dict_end, end_index, window_log);
    let dict_start = dict_base.wrapping_add(dict_start_index as usize);
    let prefix_start_index = dict_limit.max(dict_start_index);
    let prefix_start = base.wrapping_add(prefix_start_index as usize);
    let dict_end = dict_base.wrapping_add(prefix_start_index as usize);
    if prefix_start_index == dict_start_index {
        return unsafe {
            compress_block_fast_no_dict(
                hash_table,
                base,
                dict_limit,
                loaded_dict_end,
                hash_log,
                target_length,
                window_log,
                seq_store,
                reps,
                src,
                src_size,
                mls,
                window_log < 19,
            )
        };
    }

    let step_size = target_length as usize + usize::from(target_length == 0) + 1;
    let mut ip0 = istart;
    let mut anchor = istart;
    let mut offset1 = unsafe { *reps };
    let mut offset2 = unsafe { *reps.add(1) };
    let mut offset_saved1 = 0u32;
    let mut offset_saved2 = 0u32;
    let current = unsafe { index_from(base, ip0) };
    let max_rep = current.wrapping_sub(dict_start_index);
    if offset2 >= max_rep {
        offset_saved2 = offset2;
        offset2 = 0;
    }
    if offset1 >= max_rep {
        offset_saved1 = offset1;
        offset1 = 0;
    }

    'start: loop {
        let mut step = step_size;
        let mut next_step = ip0.wrapping_add(1 << (K_SEARCH_STRENGTH - 1));
        let mut ip1 = ip0.wrapping_add(1);
        let mut ip2 = ip0.wrapping_add(step);
        let mut ip3 = ip2.wrapping_add(1);
        if ptr_ge(ip3, ilimit) {
            break;
        }

        let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) };
        let mut hash1 = unsafe { hash_ptr(ip1, hash_log, mls) };
        let mut index = unsafe { table_get(hash_table, hash0) };
        let mut index_base = if index < prefix_start_index {
            dict_base
        } else {
            base
        };

        loop {
            let current2 = unsafe { index_from(base, ip2) };
            let rep_index = current2.wrapping_sub(offset1);
            let rep_base = if rep_index < prefix_start_index {
                dict_base
            } else {
                base
            };
            let rep_value = if offset1 > 0 && prefix_start_index.wrapping_sub(rep_index) >= 4 {
                unsafe { read32(rep_base.wrapping_add(rep_index as usize)) }
            } else {
                unsafe { read32(ip2) ^ 1 }
            };
            let mut current0 = unsafe { index_from(base, ip0) };
            unsafe { table_set(hash_table, hash0, current0) };

            if unsafe { read32(ip2) == rep_value } {
                ip0 = ip2;
                let mut match0 = rep_base.wrapping_add(rep_index as usize);
                let match_end = if rep_index < prefix_start_index {
                    dict_end
                } else {
                    iend
                };
                let match_length =
                    usize::from(unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) });
                ip0 = ip0.wrapping_sub(match_length);
                match0 = match0.wrapping_sub(match_length);
                let (new_ip0, new_anchor) = unsafe {
                    finish_ext_dict_match(
                        hash_table,
                        base,
                        dict_base,
                        prefix_start_index,
                        dict_end,
                        prefix_start,
                        hash_log,
                        mls,
                        seq_store,
                        &mut offset1,
                        &mut offset2,
                        ip0,
                        match0,
                        match_end,
                        anchor,
                        iend,
                        ilimit,
                        current0,
                        hash1,
                        ip1,
                        REPCODE1_TO_OFFBASE,
                        match_length + 4,
                    )
                };
                ip0 = new_ip0;
                anchor = new_anchor;
                continue 'start;
            }

            if index >= dict_start_index
                && unsafe { read32(index_base.wrapping_add(index as usize)) == read32(ip0) }
            {
                let offset = current0.wrapping_sub(index);
                let low_match_ptr = if index < prefix_start_index {
                    dict_start
                } else {
                    prefix_start
                };
                let match_end = if index < prefix_start_index {
                    dict_end
                } else {
                    iend
                };
                let mut match0 = index_base.wrapping_add(index as usize);
                offset2 = offset1;
                offset1 = offset;
                let mut match_length = 4usize;
                while ptr_gt(ip0, anchor)
                    && ptr_gt(match0, low_match_ptr)
                    && unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }
                {
                    ip0 = ip0.wrapping_sub(1);
                    match0 = match0.wrapping_sub(1);
                    match_length += 1;
                }
                let (new_ip0, new_anchor) = unsafe {
                    finish_ext_dict_match(
                        hash_table,
                        base,
                        dict_base,
                        prefix_start_index,
                        dict_end,
                        prefix_start,
                        hash_log,
                        mls,
                        seq_store,
                        &mut offset1,
                        &mut offset2,
                        ip0,
                        match0,
                        match_end,
                        anchor,
                        iend,
                        ilimit,
                        current0,
                        hash1,
                        ip1,
                        offset.wrapping_add(ZSTD_REP_NUM as u32),
                        match_length,
                    )
                };
                ip0 = new_ip0;
                anchor = new_anchor;
                continue 'start;
            }

            index = unsafe { table_get(hash_table, hash1) };
            index_base = if index < prefix_start_index {
                dict_base
            } else {
                base
            };
            hash0 = hash1;
            hash1 = unsafe { hash_ptr(ip2, hash_log, mls) };
            ip0 = ip1;
            ip1 = ip2;
            ip2 = ip3;
            current0 = unsafe { index_from(base, ip0) };
            unsafe { table_set(hash_table, hash0, current0) };

            if index >= dict_start_index
                && unsafe { read32(index_base.wrapping_add(index as usize)) == read32(ip0) }
            {
                let offset = current0.wrapping_sub(index);
                let low_match_ptr = if index < prefix_start_index {
                    dict_start
                } else {
                    prefix_start
                };
                let match_end = if index < prefix_start_index {
                    dict_end
                } else {
                    iend
                };
                let mut match0 = index_base.wrapping_add(index as usize);
                offset2 = offset1;
                offset1 = offset;
                let mut match_length = 4usize;
                while ptr_gt(ip0, anchor)
                    && ptr_gt(match0, low_match_ptr)
                    && unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }
                {
                    ip0 = ip0.wrapping_sub(1);
                    match0 = match0.wrapping_sub(1);
                    match_length += 1;
                }
                let (new_ip0, new_anchor) = unsafe {
                    finish_ext_dict_match(
                        hash_table,
                        base,
                        dict_base,
                        prefix_start_index,
                        dict_end,
                        prefix_start,
                        hash_log,
                        mls,
                        seq_store,
                        &mut offset1,
                        &mut offset2,
                        ip0,
                        match0,
                        match_end,
                        anchor,
                        iend,
                        ilimit,
                        current0,
                        hash1,
                        ip1,
                        offset.wrapping_add(ZSTD_REP_NUM as u32),
                        match_length,
                    )
                };
                ip0 = new_ip0;
                anchor = new_anchor;
                continue 'start;
            }

            index = unsafe { table_get(hash_table, hash1) };
            index_base = if index < prefix_start_index {
                dict_base
            } else {
                base
            };
            hash0 = hash1;
            hash1 = unsafe { hash_ptr(ip2, hash_log, mls) };
            ip0 = ip1;
            ip1 = ip2;
            ip2 = ip0.wrapping_add(step);
            ip3 = ip1.wrapping_add(step);
            if ptr_ge(ip2, next_step) {
                step += 1;
                next_step = next_step.wrapping_add(1 << (K_SEARCH_STRENGTH - 1));
            }
            if ptr_ge(ip3, ilimit) {
                break 'start;
            }
        }
    }

    offset_saved2 = if offset_saved1 != 0 && offset1 != 0 {
        offset_saved1
    } else {
        offset_saved2
    };
    unsafe {
        *reps = if offset1 != 0 { offset1 } else { offset_saved1 };
        *reps.add(1) = if offset2 != 0 { offset2 } else { offset_saved2 };
    }
    unsafe { iend.offset_from(anchor) as usize }
}

/// Rust implementation called by the C ABI wrapper for external dictionaries.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast_extDict(
    hash_table: *mut u32,
    base: *const u8,
    dict_base: *const u8,
    dict_limit: u32,
    low_limit: u32,
    loaded_dict_end: u32,
    hash_log: u32,
    min_match: u32,
    target_length: u32,
    window_log: u32,
    seq_store: *mut c_void,
    reps: *mut u32,
    src: *const c_void,
    src_size: usize,
) -> usize {
    unsafe {
        compress_block_fast_ext_dict(
            hash_table,
            base,
            dict_base,
            dict_limit,
            low_limit,
            loaded_dict_end,
            hash_log,
            target_length,
            window_log,
            seq_store.cast::<SeqStore_t>(),
            reps,
            src.cast::<u8>(),
            src_size,
            fast_mls(min_match),
        )
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::mem::{align_of, offset_of, size_of};

    #[test]
    fn seq_store_layout_matches_the_c_leaf_abi() {
        assert_eq!(size_of::<SeqDef>(), 8);
        assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
        assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
        assert_eq!(
            offset_of!(SeqStore_t, longLengthPos),
            9 * size_of::<usize>() + 4
        );
        assert_eq!(size_of::<SeqStore_t>(), 9 * size_of::<usize>() + 8);
    }

    #[test]
    fn tagged_index_keeps_index_and_tag_in_c_format() {
        let mut table = [0u32; 4];
        write_tagged_index(
            table.as_mut_ptr(),
            (2 << SHORT_CACHE_TAG_BITS) | 0x5a,
            0x123456,
        );
        assert_eq!(table[2], (0x123456 << SHORT_CACHE_TAG_BITS) | 0x5a);
        assert!(packed_tags_match(
            table[2],
            (1 << SHORT_CACHE_TAG_BITS) | 0x5a
        ));
        assert!(!packed_tags_match(
            table[2],
            (1 << SHORT_CACHE_TAG_BITS) | 0x5b
        ));
    }

    #[test]
    fn store_seq_preserves_literal_and_long_match_contracts() {
        let mut sequences = [SeqDef {
            offBase: 0,
            litLength: 0,
            mlBase: 0,
        }; 2];
        let mut literals = [0u8; 16];
        let source = *b"literals";
        let mut store = SeqStore_t {
            sequencesStart: sequences.as_mut_ptr(),
            sequences: sequences.as_mut_ptr(),
            litStart: literals.as_mut_ptr(),
            lit: literals.as_mut_ptr(),
            llCode: ptr::null_mut(),
            mlCode: ptr::null_mut(),
            ofCode: ptr::null_mut(),
            maxNbSeq: sequences.len(),
            maxNbLit: literals.len(),
            longLengthType: 0,
            longLengthPos: 0,
        };

        unsafe {
            store_seq_opaque(
                (&mut store as *mut SeqStore_t).cast(),
                source.len(),
                source.as_ptr().cast(),
                source.as_ptr().add(source.len()).cast(),
                13,
                3 + 0x1_0000,
            );
        }
        assert_eq!(&literals[..source.len()], &source);
        assert_eq!(sequences[0].offBase, 13);
        assert_eq!(sequences[0].litLength, source.len() as u16);
        assert_eq!(sequences[0].mlBase, 0);
        assert_eq!(store.longLengthType, 2);
        assert_eq!(store.longLengthPos, 0);
        assert_eq!(
            unsafe { store.sequences.offset_from(store.sequencesStart) },
            1
        );
    }
}