ddidderr
f9adcf6aef
Move the two-table fast match finder into Rust while retaining a small C projection of the private match-state. The Rust implementation handles normal, attached-dictionary, and external-dictionary matching without exposing the full C context ABI. The C entry points and build-time exclusion guards remain unchanged for configured consumers. Document the migrated matcher in the Rust component map and narrow the remaining-C boundary accordingly. Test Plan: - cargo test --all-targets - cargo test --target i686-unknown-linux-gnu --all-targets - cargo clippy && cargo clippy --benches && cargo clippy --tests - cargo +nightly fmt - make -B -C tests -j2 fuzzer zstreamtest invalidDictionaries poolTests - ./tests/fuzzer -s5346 -i1 --no-big-tests - ./tests/zstreamtest -i3000 -s334462 - ./tests/invalidDictionaries - timeout 20s stdbuf -oL ./tests/poolTests Refs: rust/README.md
1820 lines
60 KiB
Rust
1820 lines
60 KiB
Rust
#![allow(non_camel_case_types)]
|
|
#![allow(non_snake_case)]
|
|
#![allow(clippy::missing_safety_doc)]
|
|
#![allow(clippy::too_many_arguments)]
|
|
|
|
//! Double-fast block match finder.
|
|
//!
|
|
//! The C shim keeps `ZSTD_MatchState_t` opaque and supplies only the leaf
|
|
//! fields used by this two-table matcher. The long (8-byte) and short
|
|
//! (min-match) search loops, table fills, and sequence writes live here.
|
|
|
|
use crate::mem::{
|
|
MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_read64, MEM_readLE32, MEM_readLE64,
|
|
MEM_readST,
|
|
};
|
|
use std::ffi::c_void;
|
|
use std::mem::size_of;
|
|
use std::os::raw::c_int;
|
|
use std::ptr;
|
|
|
|
const ZSTD_REP_NUM: usize = 3;
|
|
const MINMATCH: usize = 3;
|
|
const HASH_READ_SIZE: usize = 8;
|
|
const SHORT_CACHE_TAG_BITS: u32 = 8;
|
|
const SHORT_CACHE_TAG_MASK: u32 = (1 << SHORT_CACHE_TAG_BITS) - 1;
|
|
const K_SEARCH_STRENGTH: usize = 8;
|
|
const REPCODE1_TO_OFFBASE: u32 = 1;
|
|
|
|
#[repr(C)]
|
|
#[derive(Clone, Copy)]
|
|
struct SeqDef {
|
|
offBase: u32,
|
|
litLength: u16,
|
|
mlBase: u16,
|
|
}
|
|
|
|
/// The C shim checks this leaf ABI. `ZSTD_MatchState_t` remains wholly in C.
|
|
#[repr(C)]
|
|
struct SeqStore_t {
|
|
sequencesStart: *mut SeqDef,
|
|
sequences: *mut SeqDef,
|
|
litStart: *mut u8,
|
|
lit: *mut u8,
|
|
llCode: *mut u8,
|
|
mlCode: *mut u8,
|
|
ofCode: *mut u8,
|
|
maxNbSeq: usize,
|
|
maxNbLit: usize,
|
|
longLengthType: c_int,
|
|
longLengthPos: u32,
|
|
}
|
|
|
|
#[inline]
|
|
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
|
|
(left as usize) < (right as usize)
|
|
}
|
|
|
|
#[inline]
|
|
fn ptr_le(left: *const u8, right: *const u8) -> bool {
|
|
(left as usize) <= (right as usize)
|
|
}
|
|
|
|
#[inline]
|
|
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
|
|
(left as usize) > (right as usize)
|
|
}
|
|
|
|
#[inline]
|
|
fn ptr_ge(left: *const u8, right: *const u8) -> bool {
|
|
(left as usize) >= (right as usize)
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 {
|
|
unsafe { ptr.offset_from(base) as u32 }
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn read32(ptr: *const u8) -> u32 {
|
|
unsafe { MEM_read32(ptr.cast::<c_void>()) }
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn read64(ptr: *const u8) -> u64 {
|
|
unsafe { MEM_read64(ptr.cast::<c_void>()) }
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
|
|
unsafe { *table.add(index) }
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
|
|
unsafe { *table.add(index) = value };
|
|
}
|
|
|
|
#[inline]
|
|
fn hash_shift32(value: u32, hbits: u32) -> usize {
|
|
if hbits == 0 {
|
|
0
|
|
} else {
|
|
(value >> (32 - hbits)) as usize
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
fn hash_shift64(value: u64, hbits: u32) -> usize {
|
|
if hbits == 0 {
|
|
0
|
|
} else {
|
|
(value >> (64 - hbits)) as usize
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize {
|
|
match mls {
|
|
5 => {
|
|
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
|
|
hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits)
|
|
}
|
|
6 => {
|
|
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
|
|
hash_shift64(
|
|
value.wrapping_shl(16).wrapping_mul(227_718_039_650_203),
|
|
hbits,
|
|
)
|
|
}
|
|
7 => {
|
|
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
|
|
hash_shift64(
|
|
value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627),
|
|
hbits,
|
|
)
|
|
}
|
|
8 => {
|
|
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
|
|
hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits)
|
|
}
|
|
_ => {
|
|
let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
|
|
hash_shift32(value.wrapping_mul(2_654_435_761), hbits)
|
|
}
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
fn common_bytes(word: usize) -> usize {
|
|
let zeros = if MEM_isLittleEndian() {
|
|
word.trailing_zeros()
|
|
} else {
|
|
word.leading_zeros()
|
|
};
|
|
(zeros / 8) as usize
|
|
}
|
|
|
|
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
|
|
let input_start = input;
|
|
let word_size = size_of::<usize>();
|
|
while unsafe { input_limit.offset_from(input) as usize } >= word_size {
|
|
let diff =
|
|
unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
|
|
if diff != 0 {
|
|
return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff);
|
|
}
|
|
input = input.wrapping_add(word_size);
|
|
matched = matched.wrapping_add(word_size);
|
|
}
|
|
if MEM_64bits()
|
|
&& unsafe { input_limit.offset_from(input) as usize } >= 4
|
|
&& unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
|
|
{
|
|
input = input.wrapping_add(4);
|
|
matched = matched.wrapping_add(4);
|
|
}
|
|
if unsafe { input_limit.offset_from(input) as usize } >= 2
|
|
&& unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
|
|
{
|
|
input = input.wrapping_add(2);
|
|
matched = matched.wrapping_add(2);
|
|
}
|
|
if ptr_lt(input, input_limit) && unsafe { *input == *matched } {
|
|
input = input.wrapping_add(1);
|
|
}
|
|
unsafe { input.offset_from(input_start) as usize }
|
|
}
|
|
|
|
unsafe fn count_2segments(
|
|
input: *const u8,
|
|
matched: *const u8,
|
|
input_end: *const u8,
|
|
match_end: *const u8,
|
|
input_start: *const u8,
|
|
) -> usize {
|
|
let match_remaining = unsafe { match_end.offset_from(matched) as usize };
|
|
let input_remaining = unsafe { input_end.offset_from(input) as usize };
|
|
let first_end = input.wrapping_add(match_remaining.min(input_remaining));
|
|
let first_count = unsafe { count(input, matched, first_end) };
|
|
if matched.wrapping_add(first_count) != match_end {
|
|
return first_count;
|
|
}
|
|
first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
|
|
}
|
|
|
|
#[inline]
|
|
fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
|
|
let max_distance = 1u32.wrapping_shl(window_log);
|
|
let within_window = if curr.wrapping_sub(dict_limit) > max_distance {
|
|
curr.wrapping_sub(max_distance)
|
|
} else {
|
|
dict_limit
|
|
};
|
|
if loaded_dict_end != 0 {
|
|
dict_limit
|
|
} else {
|
|
within_window
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
|
|
let max_distance = 1u32.wrapping_shl(window_log);
|
|
let within_window = if curr.wrapping_sub(low_limit) > max_distance {
|
|
curr.wrapping_sub(max_distance)
|
|
} else {
|
|
low_limit
|
|
};
|
|
if loaded_dict_end != 0 {
|
|
low_limit
|
|
} else {
|
|
within_window
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
|
|
prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
|
|
}
|
|
|
|
#[inline]
|
|
fn write_tagged_index(table: *mut u32, hash_and_tag: usize, index: u32) {
|
|
let hash = hash_and_tag >> SHORT_CACHE_TAG_BITS;
|
|
let tag = (hash_and_tag as u32) & SHORT_CACHE_TAG_MASK;
|
|
unsafe { table_set(table, hash, (index << SHORT_CACHE_TAG_BITS) | tag) };
|
|
}
|
|
|
|
#[inline]
|
|
fn packed_tags_match(first: u32, second: usize) -> bool {
|
|
(first & SHORT_CACHE_TAG_MASK) == ((second as u32) & SHORT_CACHE_TAG_MASK)
|
|
}
|
|
|
|
unsafe fn store_seq(
|
|
seq_store: *mut SeqStore_t,
|
|
lit_length: usize,
|
|
literals: *const u8,
|
|
_lit_limit: *const u8,
|
|
off_base: u32,
|
|
match_length: usize,
|
|
) {
|
|
let seq_store = unsafe { &mut *seq_store };
|
|
let sequence = seq_store.sequences;
|
|
debug_assert!(
|
|
unsafe { sequence.offset_from(seq_store.sequencesStart) as usize } < seq_store.maxNbSeq
|
|
);
|
|
debug_assert!(match_length >= MINMATCH);
|
|
if lit_length != 0 {
|
|
unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) };
|
|
}
|
|
seq_store.lit = seq_store.lit.wrapping_add(lit_length);
|
|
|
|
let sequence_index = unsafe { sequence.offset_from(seq_store.sequencesStart) as u32 };
|
|
if lit_length > u16::MAX as usize {
|
|
debug_assert_eq!(seq_store.longLengthType, 0);
|
|
seq_store.longLengthType = 1;
|
|
seq_store.longLengthPos = sequence_index;
|
|
}
|
|
unsafe { (*sequence).litLength = lit_length as u16 };
|
|
unsafe { (*sequence).offBase = off_base };
|
|
let match_base = match_length - MINMATCH;
|
|
if match_base > u16::MAX as usize {
|
|
debug_assert_eq!(seq_store.longLengthType, 0);
|
|
seq_store.longLengthType = 2;
|
|
seq_store.longLengthPos = sequence_index;
|
|
}
|
|
unsafe { (*sequence).mlBase = match_base as u16 };
|
|
seq_store.sequences = sequence.wrapping_add(1);
|
|
}
|
|
|
|
#[inline]
|
|
fn fast_mls(min_match: u32) -> u32 {
|
|
match min_match {
|
|
5..=7 => min_match,
|
|
_ => 4,
|
|
}
|
|
}
|
|
|
|
unsafe fn fill_double_hash_table(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
next_to_update: u32,
|
|
end: *const u8,
|
|
hash_log: u32,
|
|
chain_log: u32,
|
|
min_match: u32,
|
|
full_table_load: bool,
|
|
tagged_indices: bool,
|
|
) {
|
|
if unsafe { end.offset_from(base) } < HASH_READ_SIZE as isize {
|
|
return;
|
|
}
|
|
let hbits_long = hash_log
|
|
+ if tagged_indices {
|
|
SHORT_CACHE_TAG_BITS
|
|
} else {
|
|
0
|
|
};
|
|
let hbits_small = chain_log
|
|
+ if tagged_indices {
|
|
SHORT_CACHE_TAG_BITS
|
|
} else {
|
|
0
|
|
};
|
|
let mut input = base.wrapping_add(next_to_update as usize);
|
|
let input_end = end.wrapping_sub(HASH_READ_SIZE);
|
|
|
|
while ptr_le(input.wrapping_add(2), input_end) {
|
|
let current = unsafe { index_from(base, input) };
|
|
for position in 0..3usize {
|
|
let small_hash =
|
|
unsafe { hash_ptr(input.wrapping_add(position), hbits_small, min_match) };
|
|
let long_hash = unsafe { hash_ptr(input.wrapping_add(position), hbits_long, 8) };
|
|
let small_index = if tagged_indices {
|
|
small_hash >> SHORT_CACHE_TAG_BITS
|
|
} else {
|
|
small_hash
|
|
};
|
|
let long_index = if tagged_indices {
|
|
long_hash >> SHORT_CACHE_TAG_BITS
|
|
} else {
|
|
long_hash
|
|
};
|
|
if position == 0 {
|
|
if tagged_indices {
|
|
write_tagged_index(
|
|
hash_small,
|
|
small_hash,
|
|
current.wrapping_add(position as u32),
|
|
);
|
|
} else {
|
|
unsafe {
|
|
table_set(
|
|
hash_small,
|
|
small_index,
|
|
current.wrapping_add(position as u32),
|
|
)
|
|
};
|
|
}
|
|
}
|
|
if position == 0 || unsafe { table_get(hash_long, long_index) } == 0 {
|
|
if tagged_indices {
|
|
write_tagged_index(hash_long, long_hash, current.wrapping_add(position as u32));
|
|
} else {
|
|
unsafe {
|
|
table_set(hash_long, long_index, current.wrapping_add(position as u32))
|
|
};
|
|
}
|
|
}
|
|
if !full_table_load {
|
|
break;
|
|
}
|
|
}
|
|
input = input.wrapping_add(3);
|
|
}
|
|
}
|
|
|
|
/// Rust implementation called by the C ABI wrapper for `ZSTD_fillDoubleHashTable`.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_fillDoubleHashTable(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
next_to_update: u32,
|
|
end: *const c_void,
|
|
hash_log: u32,
|
|
chain_log: u32,
|
|
min_match: u32,
|
|
full_table_load: c_int,
|
|
for_cdict: c_int,
|
|
) {
|
|
unsafe {
|
|
fill_double_hash_table(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
next_to_update,
|
|
end.cast::<u8>(),
|
|
hash_log,
|
|
chain_log,
|
|
min_match,
|
|
full_table_load != 0,
|
|
for_cdict != 0,
|
|
)
|
|
};
|
|
}
|
|
|
|
unsafe fn post_no_dict_match(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
hbits_long: u32,
|
|
hbits_small: u32,
|
|
mls: u32,
|
|
seq_store: *mut SeqStore_t,
|
|
offset1: &mut u32,
|
|
offset2: &mut u32,
|
|
mut ip: *const u8,
|
|
match_length: usize,
|
|
current: u32,
|
|
iend: *const u8,
|
|
ilimit: *const u8,
|
|
) -> (*const u8, *const u8) {
|
|
ip = ip.wrapping_add(match_length);
|
|
let mut anchor = ip;
|
|
if ptr_le(ip, ilimit) {
|
|
let index_to_insert = current.wrapping_add(2);
|
|
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, index_to_insert) };
|
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
|
|
let hash = unsafe {
|
|
hash_ptr(
|
|
base.wrapping_add(index_to_insert as usize),
|
|
hbits_small,
|
|
mls,
|
|
)
|
|
};
|
|
unsafe { table_set(hash_small, hash, index_to_insert) };
|
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
|
|
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
|
|
|
|
while ptr_le(ip, ilimit)
|
|
&& *offset2 > 0
|
|
&& unsafe { read32(ip) == read32(ip.wrapping_sub(*offset2 as usize)) }
|
|
{
|
|
let repeat_length = unsafe {
|
|
count(
|
|
ip.wrapping_add(4),
|
|
ip.wrapping_add(4).wrapping_sub(*offset2 as usize),
|
|
iend,
|
|
) + 4
|
|
};
|
|
std::mem::swap(offset1, offset2);
|
|
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
|
|
unsafe { table_set(hash_small, hash, index_from(base, ip)) };
|
|
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, index_from(base, ip)) };
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
0,
|
|
anchor,
|
|
iend,
|
|
REPCODE1_TO_OFFBASE,
|
|
repeat_length,
|
|
)
|
|
};
|
|
ip = ip.wrapping_add(repeat_length);
|
|
anchor = ip;
|
|
}
|
|
}
|
|
(ip, anchor)
|
|
}
|
|
|
|
unsafe fn compress_block_double_fast_no_dict(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
dict_limit: u32,
|
|
loaded_dict_end: u32,
|
|
hash_log: u32,
|
|
chain_log: u32,
|
|
window_log: u32,
|
|
seq_store: *mut SeqStore_t,
|
|
reps: *mut u32,
|
|
src: *const u8,
|
|
src_size: usize,
|
|
mls: u32,
|
|
) -> usize {
|
|
if src_size < HASH_READ_SIZE {
|
|
return src_size;
|
|
}
|
|
let istart = src;
|
|
let iend = istart.wrapping_add(src_size);
|
|
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
|
|
let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
|
|
let prefix_lowest_index =
|
|
lowest_prefix_index(dict_limit, loaded_dict_end, end_index, window_log);
|
|
let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
|
|
let mut ip = istart;
|
|
let mut anchor = istart;
|
|
let mut offset1 = unsafe { *reps };
|
|
let mut offset2 = unsafe { *reps.add(1) };
|
|
let mut offset_saved1 = 0u32;
|
|
let mut offset_saved2 = 0u32;
|
|
|
|
if ip == prefix_lowest {
|
|
ip = ip.wrapping_add(1);
|
|
}
|
|
let current = unsafe { index_from(base, ip) };
|
|
let window_low = lowest_prefix_index(dict_limit, loaded_dict_end, current, window_log);
|
|
let max_rep = current.wrapping_sub(window_low);
|
|
if offset2 > max_rep {
|
|
offset_saved2 = offset2;
|
|
offset2 = 0;
|
|
}
|
|
if offset1 > max_rep {
|
|
offset_saved1 = offset1;
|
|
offset1 = 0;
|
|
}
|
|
|
|
'outer: loop {
|
|
let mut step = 1usize;
|
|
let mut next_step = ip.wrapping_add(1 << K_SEARCH_STRENGTH);
|
|
let mut ip1 = ip.wrapping_add(step);
|
|
if ptr_gt(ip1, ilimit) {
|
|
break;
|
|
}
|
|
|
|
let mut hash_long0 = unsafe { hash_ptr(ip, hash_log, 8) };
|
|
let mut index_long0 = unsafe { table_get(hash_long, hash_long0) };
|
|
let mut match_long0 = base.wrapping_add(index_long0 as usize);
|
|
|
|
loop {
|
|
let hash_small0 = unsafe { hash_ptr(ip, chain_log, mls) };
|
|
let index_small0 = unsafe { table_get(hash_small, hash_small0) };
|
|
let mut match_small0 = base.wrapping_add(index_small0 as usize);
|
|
let current = unsafe { index_from(base, ip) };
|
|
unsafe {
|
|
table_set(hash_long, hash_long0, current);
|
|
table_set(hash_small, hash_small0, current);
|
|
}
|
|
|
|
if offset1 > 0
|
|
&& unsafe {
|
|
read32(ip.wrapping_add(1).wrapping_sub(offset1 as usize))
|
|
== read32(ip.wrapping_add(1))
|
|
}
|
|
{
|
|
let match_length = unsafe {
|
|
count(
|
|
ip.wrapping_add(5),
|
|
ip.wrapping_add(5).wrapping_sub(offset1 as usize),
|
|
iend,
|
|
) + 4
|
|
};
|
|
ip = ip.wrapping_add(1);
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
REPCODE1_TO_OFFBASE,
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_no_dict_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue 'outer;
|
|
}
|
|
|
|
let hash_long1 = unsafe { hash_ptr(ip1, hash_log, 8) };
|
|
if index_long0 > prefix_lowest_index && unsafe { read64(match_long0) == read64(ip) } {
|
|
let mut match_length =
|
|
unsafe { count(ip.wrapping_add(8), match_long0.wrapping_add(8), iend) + 8 };
|
|
let offset = unsafe { index_from(match_long0, ip) };
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(match_long0, prefix_lowest)
|
|
&& unsafe { *ip.wrapping_sub(1) == *match_long0.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
match_long0 = match_long0.wrapping_sub(1);
|
|
match_length += 1;
|
|
}
|
|
offset2 = offset1;
|
|
offset1 = offset;
|
|
if step < 4 {
|
|
unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) };
|
|
}
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_no_dict_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue 'outer;
|
|
}
|
|
|
|
let index_long1 = unsafe { table_get(hash_long, hash_long1) };
|
|
let match_long1 = base.wrapping_add(index_long1 as usize);
|
|
if index_small0 > prefix_lowest_index && unsafe { read32(match_small0) == read32(ip) } {
|
|
let mut match_length =
|
|
unsafe { count(ip.wrapping_add(4), match_small0.wrapping_add(4), iend) + 4 };
|
|
let mut offset = unsafe { index_from(match_small0, ip) };
|
|
if index_long1 > prefix_lowest_index
|
|
&& unsafe { read64(match_long1) == read64(ip1) }
|
|
{
|
|
let long_length = unsafe {
|
|
count(ip1.wrapping_add(8), match_long1.wrapping_add(8), iend) + 8
|
|
};
|
|
if long_length > match_length {
|
|
ip = ip1;
|
|
match_length = long_length;
|
|
offset = unsafe { index_from(match_long1, ip) };
|
|
match_small0 = match_long1;
|
|
}
|
|
}
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(match_small0, prefix_lowest)
|
|
&& unsafe { *ip.wrapping_sub(1) == *match_small0.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
match_small0 = match_small0.wrapping_sub(1);
|
|
match_length += 1;
|
|
}
|
|
offset2 = offset1;
|
|
offset1 = offset;
|
|
if step < 4 {
|
|
unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) };
|
|
}
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_no_dict_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue 'outer;
|
|
}
|
|
|
|
if ptr_ge(ip1, next_step) {
|
|
step += 1;
|
|
next_step = next_step.wrapping_add(1 << K_SEARCH_STRENGTH);
|
|
}
|
|
ip = ip1;
|
|
ip1 = ip1.wrapping_add(step);
|
|
hash_long0 = hash_long1;
|
|
index_long0 = index_long1;
|
|
match_long0 = match_long1;
|
|
if ptr_gt(ip1, ilimit) {
|
|
break 'outer;
|
|
}
|
|
}
|
|
}
|
|
|
|
offset_saved2 = if offset_saved1 != 0 && offset1 != 0 {
|
|
offset_saved1
|
|
} else {
|
|
offset_saved2
|
|
};
|
|
unsafe {
|
|
*reps = if offset1 != 0 { offset1 } else { offset_saved1 };
|
|
*reps.add(1) = if offset2 != 0 { offset2 } else { offset_saved2 };
|
|
}
|
|
unsafe { iend.offset_from(anchor) as usize }
|
|
}
|
|
|
|
/// Rust implementation called by the C ABI wrapper for `ZSTD_compressBlock_doubleFast`.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
dict_limit: u32,
|
|
loaded_dict_end: u32,
|
|
hash_log: u32,
|
|
chain_log: u32,
|
|
min_match: u32,
|
|
window_log: u32,
|
|
seq_store: *mut c_void,
|
|
reps: *mut u32,
|
|
src: *const c_void,
|
|
src_size: usize,
|
|
) -> usize {
|
|
unsafe {
|
|
compress_block_double_fast_no_dict(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_limit,
|
|
loaded_dict_end,
|
|
hash_log,
|
|
chain_log,
|
|
window_log,
|
|
seq_store.cast::<SeqStore_t>(),
|
|
reps,
|
|
src.cast::<u8>(),
|
|
src_size,
|
|
fast_mls(min_match),
|
|
)
|
|
}
|
|
}
|
|
|
|
unsafe fn post_dict_match(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
dict_base: *const u8,
|
|
prefix_lowest_index: u32,
|
|
dict_index_delta: u32,
|
|
dict_end: *const u8,
|
|
prefix_lowest: *const u8,
|
|
hbits_long: u32,
|
|
hbits_small: u32,
|
|
mls: u32,
|
|
seq_store: *mut SeqStore_t,
|
|
offset1: &mut u32,
|
|
offset2: &mut u32,
|
|
mut ip: *const u8,
|
|
match_length: usize,
|
|
current: u32,
|
|
iend: *const u8,
|
|
ilimit: *const u8,
|
|
) -> (*const u8, *const u8) {
|
|
ip = ip.wrapping_add(match_length);
|
|
let mut anchor = ip;
|
|
if ptr_le(ip, ilimit) {
|
|
let index_to_insert = current.wrapping_add(2);
|
|
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, index_to_insert) };
|
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
|
|
let hash = unsafe {
|
|
hash_ptr(
|
|
base.wrapping_add(index_to_insert as usize),
|
|
hbits_small,
|
|
mls,
|
|
)
|
|
};
|
|
unsafe { table_set(hash_small, hash, index_to_insert) };
|
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
|
|
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
|
|
|
|
while ptr_le(ip, ilimit) {
|
|
let current2 = unsafe { index_from(base, ip) };
|
|
let rep_index2 = current2.wrapping_sub(*offset2);
|
|
let rep_match2 = if rep_index2 < prefix_lowest_index {
|
|
dict_base
|
|
.wrapping_sub(dict_index_delta as usize)
|
|
.wrapping_add(rep_index2 as usize)
|
|
} else {
|
|
base.wrapping_add(rep_index2 as usize)
|
|
};
|
|
if index_overlap_check(prefix_lowest_index, rep_index2)
|
|
&& unsafe { read32(rep_match2) == read32(ip) }
|
|
{
|
|
let rep_end2 = if rep_index2 < prefix_lowest_index {
|
|
dict_end
|
|
} else {
|
|
iend
|
|
};
|
|
let repeat_length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(4),
|
|
rep_match2.wrapping_add(4),
|
|
iend,
|
|
rep_end2,
|
|
prefix_lowest,
|
|
) + 4
|
|
};
|
|
std::mem::swap(offset1, offset2);
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
0,
|
|
anchor,
|
|
iend,
|
|
REPCODE1_TO_OFFBASE,
|
|
repeat_length,
|
|
)
|
|
};
|
|
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
|
|
unsafe { table_set(hash_small, hash, current2) };
|
|
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, current2) };
|
|
ip = ip.wrapping_add(repeat_length);
|
|
anchor = ip;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
(ip, anchor)
|
|
}
|
|
|
|
unsafe fn compress_block_double_fast_dict_match_state(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
prefix_lowest_index: u32,
|
|
hash_log: u32,
|
|
chain_log: u32,
|
|
seq_store: *mut SeqStore_t,
|
|
reps: *mut u32,
|
|
src: *const u8,
|
|
src_size: usize,
|
|
mls: u32,
|
|
dict_hash_long: *const u32,
|
|
dict_hash_small: *const u32,
|
|
dict_base: *const u8,
|
|
dict_start_index: u32,
|
|
dict_end: *const u8,
|
|
dict_hash_log: u32,
|
|
dict_chain_log: u32,
|
|
_prefetch_cdict_tables: bool,
|
|
) -> usize {
|
|
if src_size < HASH_READ_SIZE {
|
|
return src_size;
|
|
}
|
|
let istart = src;
|
|
let iend = istart.wrapping_add(src_size);
|
|
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
|
|
let mut ip = istart;
|
|
let mut anchor = istart;
|
|
let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
|
|
let dict_start = dict_base.wrapping_add(dict_start_index as usize);
|
|
let dict_index_delta =
|
|
prefix_lowest_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) });
|
|
let dict_hbits_long = dict_hash_log + SHORT_CACHE_TAG_BITS;
|
|
let dict_hbits_small = dict_chain_log + SHORT_CACHE_TAG_BITS;
|
|
let dict_and_prefix_length =
|
|
unsafe { ip.offset_from(prefix_lowest) as u32 + dict_end.offset_from(dict_start) as u32 };
|
|
let mut offset1 = unsafe { *reps };
|
|
let mut offset2 = unsafe { *reps.add(1) };
|
|
if dict_and_prefix_length == 0 {
|
|
ip = ip.wrapping_add(1);
|
|
}
|
|
|
|
'outer: while ptr_lt(ip, ilimit) {
|
|
let h_long = unsafe { hash_ptr(ip, hash_log, 8) };
|
|
let h_small = unsafe { hash_ptr(ip, chain_log, mls) };
|
|
let dict_hash_and_tag_long = unsafe { hash_ptr(ip, dict_hbits_long, 8) };
|
|
let dict_hash_and_tag_small = unsafe { hash_ptr(ip, dict_hbits_small, mls) };
|
|
let dict_match_index_and_tag_long = unsafe {
|
|
table_get(
|
|
dict_hash_long,
|
|
dict_hash_and_tag_long >> SHORT_CACHE_TAG_BITS,
|
|
)
|
|
};
|
|
let dict_match_index_and_tag_small = unsafe {
|
|
table_get(
|
|
dict_hash_small,
|
|
dict_hash_and_tag_small >> SHORT_CACHE_TAG_BITS,
|
|
)
|
|
};
|
|
let dict_tags_match_long =
|
|
packed_tags_match(dict_match_index_and_tag_long, dict_hash_and_tag_long);
|
|
let dict_tags_match_small =
|
|
packed_tags_match(dict_match_index_and_tag_small, dict_hash_and_tag_small);
|
|
let current = unsafe { index_from(base, ip) };
|
|
let match_index_long = unsafe { table_get(hash_long, h_long) };
|
|
let mut match_index_small = unsafe { table_get(hash_small, h_small) };
|
|
let mut match_long = base.wrapping_add(match_index_long as usize);
|
|
let mut matched = base.wrapping_add(match_index_small as usize);
|
|
let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
|
|
let rep_match = if rep_index < prefix_lowest_index {
|
|
dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
|
|
} else {
|
|
base.wrapping_add(rep_index as usize)
|
|
};
|
|
unsafe {
|
|
table_set(hash_long, h_long, current);
|
|
table_set(hash_small, h_small, current);
|
|
}
|
|
|
|
if index_overlap_check(prefix_lowest_index, rep_index)
|
|
&& unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
|
|
{
|
|
let rep_match_end = if rep_index < prefix_lowest_index {
|
|
dict_end
|
|
} else {
|
|
iend
|
|
};
|
|
let match_length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(5),
|
|
rep_match.wrapping_add(4),
|
|
iend,
|
|
rep_match_end,
|
|
prefix_lowest,
|
|
) + 4
|
|
};
|
|
ip = ip.wrapping_add(1);
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
REPCODE1_TO_OFFBASE,
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_dict_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_base,
|
|
prefix_lowest_index,
|
|
dict_index_delta,
|
|
dict_end,
|
|
prefix_lowest,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue;
|
|
}
|
|
|
|
if match_index_long >= prefix_lowest_index && unsafe { read64(match_long) == read64(ip) } {
|
|
let mut match_length =
|
|
unsafe { count(ip.wrapping_add(8), match_long.wrapping_add(8), iend) + 8 };
|
|
let offset = unsafe { index_from(match_long, ip) };
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(match_long, prefix_lowest)
|
|
&& unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
match_long = match_long.wrapping_sub(1);
|
|
match_length += 1;
|
|
}
|
|
offset2 = offset1;
|
|
offset1 = offset;
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_dict_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_base,
|
|
prefix_lowest_index,
|
|
dict_index_delta,
|
|
dict_end,
|
|
prefix_lowest,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue;
|
|
}
|
|
|
|
if dict_tags_match_long {
|
|
let dict_match_index = dict_match_index_and_tag_long >> SHORT_CACHE_TAG_BITS;
|
|
let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
|
|
if ptr_gt(dict_match, dict_start) && unsafe { read64(dict_match) == read64(ip) } {
|
|
let mut match_length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(8),
|
|
dict_match.wrapping_add(8),
|
|
iend,
|
|
dict_end,
|
|
prefix_lowest,
|
|
) + 8
|
|
};
|
|
let offset = current
|
|
.wrapping_sub(dict_match_index)
|
|
.wrapping_sub(dict_index_delta);
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(dict_match, dict_start)
|
|
&& unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
dict_match = dict_match.wrapping_sub(1);
|
|
match_length += 1;
|
|
}
|
|
offset2 = offset1;
|
|
offset1 = offset;
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_dict_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_base,
|
|
prefix_lowest_index,
|
|
dict_index_delta,
|
|
dict_end,
|
|
prefix_lowest,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
let mut short_found = false;
|
|
if match_index_small > prefix_lowest_index {
|
|
short_found = unsafe { read32(matched) == read32(ip) };
|
|
} else if dict_tags_match_small {
|
|
let dict_match_index = dict_match_index_and_tag_small >> SHORT_CACHE_TAG_BITS;
|
|
let dict_match = dict_base.wrapping_add(dict_match_index as usize);
|
|
match_index_small = dict_match_index.wrapping_add(dict_index_delta);
|
|
if ptr_gt(dict_match, dict_start) && unsafe { read32(dict_match) == read32(ip) } {
|
|
matched = dict_match;
|
|
short_found = true;
|
|
}
|
|
}
|
|
if !short_found {
|
|
ip = ip.wrapping_add(
|
|
(unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1,
|
|
);
|
|
continue;
|
|
}
|
|
|
|
let hash_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) };
|
|
let dict_hash_and_tag_long3 = unsafe { hash_ptr(ip.wrapping_add(1), dict_hbits_long, 8) };
|
|
let match_index_long3 = unsafe { table_get(hash_long, hash_long3) };
|
|
let dict_match_index_and_tag_long3 = unsafe {
|
|
table_get(
|
|
dict_hash_long,
|
|
dict_hash_and_tag_long3 >> SHORT_CACHE_TAG_BITS,
|
|
)
|
|
};
|
|
let dict_tags_match_long3 =
|
|
packed_tags_match(dict_match_index_and_tag_long3, dict_hash_and_tag_long3);
|
|
let mut match_long3 = base.wrapping_add(match_index_long3 as usize);
|
|
unsafe { table_set(hash_long, hash_long3, current.wrapping_add(1)) };
|
|
|
|
let (match_length, offset) = if match_index_long3 >= prefix_lowest_index
|
|
&& unsafe { read64(match_long3) == read64(ip.wrapping_add(1)) }
|
|
{
|
|
let mut length =
|
|
unsafe { count(ip.wrapping_add(9), match_long3.wrapping_add(8), iend) + 8 };
|
|
ip = ip.wrapping_add(1);
|
|
let offset = unsafe { index_from(match_long3, ip) };
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(match_long3, prefix_lowest)
|
|
&& unsafe { *ip.wrapping_sub(1) == *match_long3.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
match_long3 = match_long3.wrapping_sub(1);
|
|
length += 1;
|
|
}
|
|
(length, offset)
|
|
} else if dict_tags_match_long3 {
|
|
let dict_match_index = dict_match_index_and_tag_long3 >> SHORT_CACHE_TAG_BITS;
|
|
let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
|
|
if ptr_gt(dict_match, dict_start)
|
|
&& unsafe { read64(dict_match) == read64(ip.wrapping_add(1)) }
|
|
{
|
|
let mut length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(9),
|
|
dict_match.wrapping_add(8),
|
|
iend,
|
|
dict_end,
|
|
prefix_lowest,
|
|
) + 8
|
|
};
|
|
ip = ip.wrapping_add(1);
|
|
let offset = current
|
|
.wrapping_add(1)
|
|
.wrapping_sub(dict_match_index)
|
|
.wrapping_sub(dict_index_delta);
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(dict_match, dict_start)
|
|
&& unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
dict_match = dict_match.wrapping_sub(1);
|
|
length += 1;
|
|
}
|
|
(length, offset)
|
|
} else if match_index_small < prefix_lowest_index {
|
|
let mut length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(4),
|
|
matched.wrapping_add(4),
|
|
iend,
|
|
dict_end,
|
|
prefix_lowest,
|
|
) + 4
|
|
};
|
|
let offset = current.wrapping_sub(match_index_small);
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(matched, dict_start)
|
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
matched = matched.wrapping_sub(1);
|
|
length += 1;
|
|
}
|
|
(length, offset)
|
|
} else {
|
|
let mut length =
|
|
unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 };
|
|
let offset = unsafe { index_from(matched, ip) };
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(matched, prefix_lowest)
|
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
matched = matched.wrapping_sub(1);
|
|
length += 1;
|
|
}
|
|
(length, offset)
|
|
}
|
|
} else if match_index_small < prefix_lowest_index {
|
|
let mut length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(4),
|
|
matched.wrapping_add(4),
|
|
iend,
|
|
dict_end,
|
|
prefix_lowest,
|
|
) + 4
|
|
};
|
|
let offset = current.wrapping_sub(match_index_small);
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(matched, dict_start)
|
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
matched = matched.wrapping_sub(1);
|
|
length += 1;
|
|
}
|
|
(length, offset)
|
|
} else {
|
|
let mut length =
|
|
unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 };
|
|
let offset = unsafe { index_from(matched, ip) };
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(matched, prefix_lowest)
|
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
matched = matched.wrapping_sub(1);
|
|
length += 1;
|
|
}
|
|
(length, offset)
|
|
};
|
|
offset2 = offset1;
|
|
offset1 = offset;
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_dict_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_base,
|
|
prefix_lowest_index,
|
|
dict_index_delta,
|
|
dict_end,
|
|
prefix_lowest,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue 'outer;
|
|
}
|
|
|
|
unsafe {
|
|
*reps = offset1;
|
|
*reps.add(1) = offset2;
|
|
}
|
|
unsafe { iend.offset_from(anchor) as usize }
|
|
}
|
|
|
|
/// Rust implementation called by the C ABI wrapper for attached dictionaries.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_dictMatchState(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
prefix_lowest_index: u32,
|
|
hash_log: u32,
|
|
chain_log: u32,
|
|
min_match: u32,
|
|
seq_store: *mut c_void,
|
|
reps: *mut u32,
|
|
src: *const c_void,
|
|
src_size: usize,
|
|
dict_hash_long: *const u32,
|
|
dict_hash_small: *const u32,
|
|
dict_base: *const u8,
|
|
dict_start_index: u32,
|
|
dict_end: *const u8,
|
|
dict_hash_log: u32,
|
|
dict_chain_log: u32,
|
|
prefetch_cdict_tables: c_int,
|
|
) -> usize {
|
|
unsafe {
|
|
compress_block_double_fast_dict_match_state(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
prefix_lowest_index,
|
|
hash_log,
|
|
chain_log,
|
|
seq_store.cast::<SeqStore_t>(),
|
|
reps,
|
|
src.cast::<u8>(),
|
|
src_size,
|
|
fast_mls(min_match),
|
|
dict_hash_long,
|
|
dict_hash_small,
|
|
dict_base,
|
|
dict_start_index,
|
|
dict_end,
|
|
dict_hash_log,
|
|
dict_chain_log,
|
|
prefetch_cdict_tables != 0,
|
|
)
|
|
}
|
|
}
|
|
|
|
unsafe fn post_ext_match(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
dict_base: *const u8,
|
|
dict_start_index: u32,
|
|
prefix_start_index: u32,
|
|
dict_end: *const u8,
|
|
prefix_start: *const u8,
|
|
hbits_long: u32,
|
|
hbits_small: u32,
|
|
mls: u32,
|
|
seq_store: *mut SeqStore_t,
|
|
offset1: &mut u32,
|
|
offset2: &mut u32,
|
|
mut ip: *const u8,
|
|
match_length: usize,
|
|
current: u32,
|
|
iend: *const u8,
|
|
ilimit: *const u8,
|
|
) -> (*const u8, *const u8) {
|
|
ip = ip.wrapping_add(match_length);
|
|
let mut anchor = ip;
|
|
if ptr_le(ip, ilimit) {
|
|
let index_to_insert = current.wrapping_add(2);
|
|
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, index_to_insert) };
|
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
|
|
let hash = unsafe {
|
|
hash_ptr(
|
|
base.wrapping_add(index_to_insert as usize),
|
|
hbits_small,
|
|
mls,
|
|
)
|
|
};
|
|
unsafe { table_set(hash_small, hash, index_to_insert) };
|
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
|
|
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
|
|
|
|
while ptr_le(ip, ilimit) {
|
|
let current2 = unsafe { index_from(base, ip) };
|
|
let rep_index2 = current2.wrapping_sub(*offset2);
|
|
let rep_match2 = if rep_index2 < prefix_start_index {
|
|
dict_base.wrapping_add(rep_index2 as usize)
|
|
} else {
|
|
base.wrapping_add(rep_index2 as usize)
|
|
};
|
|
if index_overlap_check(prefix_start_index, rep_index2)
|
|
&& *offset2 <= current2.wrapping_sub(dict_start_index)
|
|
&& unsafe { read32(rep_match2) == read32(ip) }
|
|
{
|
|
let rep_end2 = if rep_index2 < prefix_start_index {
|
|
dict_end
|
|
} else {
|
|
iend
|
|
};
|
|
let repeat_length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(4),
|
|
rep_match2.wrapping_add(4),
|
|
iend,
|
|
rep_end2,
|
|
prefix_start,
|
|
) + 4
|
|
};
|
|
std::mem::swap(offset1, offset2);
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
0,
|
|
anchor,
|
|
iend,
|
|
REPCODE1_TO_OFFBASE,
|
|
repeat_length,
|
|
)
|
|
};
|
|
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
|
|
unsafe { table_set(hash_small, hash, current2) };
|
|
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
|
|
unsafe { table_set(hash_long, hash, current2) };
|
|
ip = ip.wrapping_add(repeat_length);
|
|
anchor = ip;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
(ip, anchor)
|
|
}
|
|
|
|
unsafe fn compress_block_double_fast_ext_dict(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
dict_base: *const u8,
|
|
dict_limit: u32,
|
|
low_limit: u32,
|
|
loaded_dict_end: u32,
|
|
hash_log: u32,
|
|
chain_log: u32,
|
|
window_log: u32,
|
|
seq_store: *mut SeqStore_t,
|
|
reps: *mut u32,
|
|
src: *const u8,
|
|
src_size: usize,
|
|
mls: u32,
|
|
) -> usize {
|
|
if src_size < HASH_READ_SIZE {
|
|
return src_size;
|
|
}
|
|
let istart = src;
|
|
let iend = istart.wrapping_add(src_size);
|
|
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
|
|
let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
|
|
let dict_start_index = lowest_match_index(low_limit, loaded_dict_end, end_index, window_log);
|
|
let prefix_start_index = dict_limit.max(dict_start_index);
|
|
if prefix_start_index == dict_start_index {
|
|
return unsafe {
|
|
compress_block_double_fast_no_dict(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_limit,
|
|
loaded_dict_end,
|
|
hash_log,
|
|
chain_log,
|
|
window_log,
|
|
seq_store,
|
|
reps,
|
|
src,
|
|
src_size,
|
|
mls,
|
|
)
|
|
};
|
|
}
|
|
let prefix_start = base.wrapping_add(prefix_start_index as usize);
|
|
let dict_start = dict_base.wrapping_add(dict_start_index as usize);
|
|
let dict_end = dict_base.wrapping_add(prefix_start_index as usize);
|
|
let mut ip = istart;
|
|
let mut anchor = istart;
|
|
let mut offset1 = unsafe { *reps };
|
|
let mut offset2 = unsafe { *reps.add(1) };
|
|
|
|
while ptr_lt(ip, ilimit) {
|
|
let h_small = unsafe { hash_ptr(ip, chain_log, mls) };
|
|
let match_index = unsafe { table_get(hash_small, h_small) };
|
|
let match_base = if match_index < prefix_start_index {
|
|
dict_base
|
|
} else {
|
|
base
|
|
};
|
|
let mut matched = match_base.wrapping_add(match_index as usize);
|
|
let h_long = unsafe { hash_ptr(ip, hash_log, 8) };
|
|
let match_long_index = unsafe { table_get(hash_long, h_long) };
|
|
let match_long_base = if match_long_index < prefix_start_index {
|
|
dict_base
|
|
} else {
|
|
base
|
|
};
|
|
let mut match_long = match_long_base.wrapping_add(match_long_index as usize);
|
|
let current = unsafe { index_from(base, ip) };
|
|
let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
|
|
let rep_base = if rep_index < prefix_start_index {
|
|
dict_base
|
|
} else {
|
|
base
|
|
};
|
|
let rep_match = rep_base.wrapping_add(rep_index as usize);
|
|
unsafe {
|
|
table_set(hash_small, h_small, current);
|
|
table_set(hash_long, h_long, current);
|
|
}
|
|
|
|
if index_overlap_check(prefix_start_index, rep_index)
|
|
&& offset1 <= current.wrapping_add(1).wrapping_sub(dict_start_index)
|
|
&& unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
|
|
{
|
|
let rep_end = if rep_index < prefix_start_index {
|
|
dict_end
|
|
} else {
|
|
iend
|
|
};
|
|
let match_length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(5),
|
|
rep_match.wrapping_add(4),
|
|
iend,
|
|
rep_end,
|
|
prefix_start,
|
|
) + 4
|
|
};
|
|
ip = ip.wrapping_add(1);
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
REPCODE1_TO_OFFBASE,
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_ext_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_base,
|
|
dict_start_index,
|
|
prefix_start_index,
|
|
dict_end,
|
|
prefix_start,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue;
|
|
}
|
|
|
|
if match_long_index > dict_start_index && unsafe { read64(match_long) == read64(ip) } {
|
|
let match_end = if match_long_index < prefix_start_index {
|
|
dict_end
|
|
} else {
|
|
iend
|
|
};
|
|
let low_match = if match_long_index < prefix_start_index {
|
|
dict_start
|
|
} else {
|
|
prefix_start
|
|
};
|
|
let mut match_length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(8),
|
|
match_long.wrapping_add(8),
|
|
iend,
|
|
match_end,
|
|
prefix_start,
|
|
) + 8
|
|
};
|
|
let offset = current.wrapping_sub(match_long_index);
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(match_long, low_match)
|
|
&& unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
match_long = match_long.wrapping_sub(1);
|
|
match_length += 1;
|
|
}
|
|
offset2 = offset1;
|
|
offset1 = offset;
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_ext_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_base,
|
|
dict_start_index,
|
|
prefix_start_index,
|
|
dict_end,
|
|
prefix_start,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue;
|
|
}
|
|
|
|
if match_index > dict_start_index && unsafe { read32(matched) == read32(ip) } {
|
|
let h_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) };
|
|
let match_index3 = unsafe { table_get(hash_long, h_long3) };
|
|
let match_base3 = if match_index3 < prefix_start_index {
|
|
dict_base
|
|
} else {
|
|
base
|
|
};
|
|
let mut match3 = match_base3.wrapping_add(match_index3 as usize);
|
|
unsafe { table_set(hash_long, h_long3, current.wrapping_add(1)) };
|
|
let (match_length, offset) = if match_index3 > dict_start_index
|
|
&& unsafe { read64(match3) == read64(ip.wrapping_add(1)) }
|
|
{
|
|
let match_end = if match_index3 < prefix_start_index {
|
|
dict_end
|
|
} else {
|
|
iend
|
|
};
|
|
let low_match = if match_index3 < prefix_start_index {
|
|
dict_start
|
|
} else {
|
|
prefix_start
|
|
};
|
|
let mut length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(9),
|
|
match3.wrapping_add(8),
|
|
iend,
|
|
match_end,
|
|
prefix_start,
|
|
) + 8
|
|
};
|
|
ip = ip.wrapping_add(1);
|
|
let offset = current.wrapping_add(1).wrapping_sub(match_index3);
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(match3, low_match)
|
|
&& unsafe { *ip.wrapping_sub(1) == *match3.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
match3 = match3.wrapping_sub(1);
|
|
length += 1;
|
|
}
|
|
(length, offset)
|
|
} else {
|
|
let match_end = if match_index < prefix_start_index {
|
|
dict_end
|
|
} else {
|
|
iend
|
|
};
|
|
let low_match = if match_index < prefix_start_index {
|
|
dict_start
|
|
} else {
|
|
prefix_start
|
|
};
|
|
let mut length = unsafe {
|
|
count_2segments(
|
|
ip.wrapping_add(4),
|
|
matched.wrapping_add(4),
|
|
iend,
|
|
match_end,
|
|
prefix_start,
|
|
) + 4
|
|
};
|
|
let offset = current.wrapping_sub(match_index);
|
|
while ptr_gt(ip, anchor)
|
|
&& ptr_gt(matched, low_match)
|
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
|
{
|
|
ip = ip.wrapping_sub(1);
|
|
matched = matched.wrapping_sub(1);
|
|
length += 1;
|
|
}
|
|
(length, offset)
|
|
};
|
|
offset2 = offset1;
|
|
offset1 = offset;
|
|
unsafe {
|
|
store_seq(
|
|
seq_store,
|
|
ip.offset_from(anchor) as usize,
|
|
anchor,
|
|
iend,
|
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
|
match_length,
|
|
)
|
|
};
|
|
let (next_ip, next_anchor) = unsafe {
|
|
post_ext_match(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_base,
|
|
dict_start_index,
|
|
prefix_start_index,
|
|
dict_end,
|
|
prefix_start,
|
|
hash_log,
|
|
chain_log,
|
|
mls,
|
|
seq_store,
|
|
&mut offset1,
|
|
&mut offset2,
|
|
ip,
|
|
match_length,
|
|
current,
|
|
iend,
|
|
ilimit,
|
|
)
|
|
};
|
|
ip = next_ip;
|
|
anchor = next_anchor;
|
|
continue;
|
|
}
|
|
|
|
ip = ip.wrapping_add((unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1);
|
|
}
|
|
|
|
unsafe {
|
|
*reps = offset1;
|
|
*reps.add(1) = offset2;
|
|
}
|
|
unsafe { iend.offset_from(anchor) as usize }
|
|
}
|
|
|
|
/// Rust implementation called by the C ABI wrapper for external dictionaries.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_extDict(
|
|
hash_long: *mut u32,
|
|
hash_small: *mut u32,
|
|
base: *const u8,
|
|
dict_base: *const u8,
|
|
dict_limit: u32,
|
|
low_limit: u32,
|
|
loaded_dict_end: u32,
|
|
hash_log: u32,
|
|
chain_log: u32,
|
|
min_match: u32,
|
|
window_log: u32,
|
|
seq_store: *mut c_void,
|
|
reps: *mut u32,
|
|
src: *const c_void,
|
|
src_size: usize,
|
|
) -> usize {
|
|
unsafe {
|
|
compress_block_double_fast_ext_dict(
|
|
hash_long,
|
|
hash_small,
|
|
base,
|
|
dict_base,
|
|
dict_limit,
|
|
low_limit,
|
|
loaded_dict_end,
|
|
hash_log,
|
|
chain_log,
|
|
window_log,
|
|
seq_store.cast::<SeqStore_t>(),
|
|
reps,
|
|
src.cast::<u8>(),
|
|
src_size,
|
|
fast_mls(min_match),
|
|
)
|
|
}
|
|
}
|