feat(compress): port the optimal parser to Rust
Move the binary-tree optimal and ultra block parsers into Rust with a narrow C projection of match-state, sequence-store, and entropy-table fields. Keep the established C entry points as wrappers, and avoid reading uninitialized entropy costs during dictionary tree updates. Test Plan: - cargo test --manifest-path rust/Cargo.toml - cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check - make -B -C tests test-zstream V=1 - git diff --check Refs: rust/src/zstd_opt.rs, lib/compress/zstd_opt.c
This commit is contained in:
@@ -55,6 +55,8 @@ pub mod zstd_lazy;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_ldm;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_opt;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_opt_tree;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_presplit;
|
||||
|
||||
@@ -0,0 +1,1836 @@
|
||||
#![allow(non_camel_case_types)]
|
||||
#![allow(non_snake_case)]
|
||||
#![allow(clippy::missing_safety_doc)]
|
||||
#![allow(clippy::too_many_arguments)]
|
||||
#![allow(clippy::not_unsafe_ptr_arg_deref)]
|
||||
|
||||
//! Optimal block parsing for the binary-tree compression strategies.
|
||||
//!
|
||||
//! The compression context remains private to C. `zstd_opt.c` projects the
|
||||
//! fields used by this module into `ZSTD_RustOptState`; all buffers and tables
|
||||
//! remain owned by the caller. This keeps the ABI independent of the private
|
||||
//! `ZSTD_MatchState_t` layout while moving the price model, match enumeration,
|
||||
//! optimal parse, and sequence emission into Rust.
|
||||
|
||||
use crate::bits::ZSTD_highbit32;
|
||||
use crate::common::{LL_BITS, MAX_LIT, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS};
|
||||
use crate::mem::{
|
||||
MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST,
|
||||
};
|
||||
use std::cmp::min;
|
||||
use std::ffi::c_void;
|
||||
use std::mem::size_of;
|
||||
use std::os::raw::{c_int, c_uint};
|
||||
use std::ptr;
|
||||
|
||||
const ZSTD_REP_NUM: usize = 3;
|
||||
const HASH_READ_SIZE: usize = 8;
|
||||
const ZSTD_OPT_NUM: u32 = 1 << 12;
|
||||
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
|
||||
const ZSTD_LITFREQ_ADD: u32 = 2;
|
||||
const ZSTD_MAX_PRICE: i32 = 1 << 30;
|
||||
const ZSTD_PREDEF_THRESHOLD: usize = 8;
|
||||
const BITCOST_ACCURACY: u32 = 8;
|
||||
const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY;
|
||||
const OFFSET_OFFBASE: u32 = ZSTD_REP_NUM as u32;
|
||||
const PRIME3BYTES: u32 = 506_832_829;
|
||||
|
||||
const ZOP_DYNAMIC: c_int = 0;
|
||||
const ZOP_PREDEF: c_int = 1;
|
||||
const ZSTD_PS_DISABLE: c_int = 2;
|
||||
|
||||
const DICT_NO_DICT: c_int = 0;
|
||||
const DICT_EXT: c_int = 1;
|
||||
const DICT_MATCH_STATE: c_int = 2;
|
||||
|
||||
const LL_CODE: [u8; 64] = [
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20,
|
||||
20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
];
|
||||
|
||||
#[allow(dead_code)]
|
||||
const ML_CODE: [u8; 149] = [
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
|
||||
26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38,
|
||||
38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42,
|
||||
];
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
struct SeqDef {
|
||||
offBase: u32,
|
||||
litLength: u16,
|
||||
mlBase: u16,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct SeqStore_t {
|
||||
sequencesStart: *mut SeqDef,
|
||||
sequences: *mut SeqDef,
|
||||
litStart: *mut u8,
|
||||
lit: *mut u8,
|
||||
llCode: *mut u8,
|
||||
mlCode: *mut u8,
|
||||
ofCode: *mut u8,
|
||||
maxNbSeq: usize,
|
||||
maxNbLit: usize,
|
||||
longLengthType: c_int,
|
||||
longLengthPos: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct ZSTD_match_t {
|
||||
off: u32,
|
||||
len: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct ZSTD_optimal_t {
|
||||
price: c_int,
|
||||
off: u32,
|
||||
mlen: u32,
|
||||
litlen: u32,
|
||||
rep: [u32; ZSTD_REP_NUM],
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct RawSeq {
|
||||
offset: u32,
|
||||
litLength: u32,
|
||||
matchLength: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
struct RawSeqStore {
|
||||
seq: *const RawSeq,
|
||||
pos: usize,
|
||||
posInSequence: usize,
|
||||
size: usize,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl Default for RawSeqStore {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
seq: ptr::null(),
|
||||
pos: 0,
|
||||
posInSequence: 0,
|
||||
size: 0,
|
||||
capacity: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct OptState {
|
||||
litFreq: *mut c_uint,
|
||||
litLengthFreq: *mut c_uint,
|
||||
matchLengthFreq: *mut c_uint,
|
||||
offCodeFreq: *mut c_uint,
|
||||
matchTable: *mut ZSTD_match_t,
|
||||
priceTable: *mut ZSTD_optimal_t,
|
||||
litSum: u32,
|
||||
litLengthSum: u32,
|
||||
matchLengthSum: u32,
|
||||
offCodeSum: u32,
|
||||
litSumBasePrice: u32,
|
||||
litLengthSumBasePrice: u32,
|
||||
matchLengthSumBasePrice: u32,
|
||||
offCodeSumBasePrice: u32,
|
||||
priceType: c_int,
|
||||
symbolCosts: *const c_void,
|
||||
literalCompressionMode: c_int,
|
||||
}
|
||||
|
||||
/* The first fields intentionally match zstd_opt_tree's public leaf view. */
|
||||
#[repr(C)]
|
||||
struct OptTreePrefix {
|
||||
hash_table: *mut u32,
|
||||
chain_table: *mut u32,
|
||||
base: *const u8,
|
||||
dict_base: *const u8,
|
||||
dict_limit: u32,
|
||||
low_limit: u32,
|
||||
loaded_dict_end: u32,
|
||||
next_to_update: *mut u32,
|
||||
hash_log: u32,
|
||||
chain_log: u32,
|
||||
search_log: u32,
|
||||
window_log: u32,
|
||||
use_c_predict: c_int,
|
||||
}
|
||||
|
||||
/// Private-state projection populated by `lib/compress/zstd_opt.c`.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_RustOptState {
|
||||
hash_table: *mut u32,
|
||||
chain_table: *mut u32,
|
||||
base: *const u8,
|
||||
dict_base: *const u8,
|
||||
dict_limit: u32,
|
||||
low_limit: u32,
|
||||
loaded_dict_end: u32,
|
||||
next_to_update: *mut u32,
|
||||
hash_log: u32,
|
||||
chain_log: u32,
|
||||
search_log: u32,
|
||||
window_log: u32,
|
||||
use_c_predict: c_int,
|
||||
hash_table3: *mut u32,
|
||||
hash_log3: u32,
|
||||
next_src: *const u8,
|
||||
min_match: u32,
|
||||
target_length: u32,
|
||||
opt: *mut OptState,
|
||||
dict_match_state: *const ZSTD_RustOptState,
|
||||
ldm_seq_store: *const RawSeqStore,
|
||||
window_base: *mut *const u8,
|
||||
window_dict_limit: *mut u32,
|
||||
window_low_limit: *mut u32,
|
||||
huf_ctable: *const usize,
|
||||
huf_repeat_valid: c_int,
|
||||
fse_litlength_ctable: *const u32,
|
||||
fse_matchlength_ctable: *const u32,
|
||||
fse_offcode_ctable: *const u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
struct FseSymbolCompressionTransform {
|
||||
deltaFindState: c_int,
|
||||
deltaNbBits: u32,
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
fn ZSTD_rust_opt_updateTreeInternal(
|
||||
state: *mut OptTreePrefix,
|
||||
ip: *const c_void,
|
||||
iend: *const c_void,
|
||||
mls: u32,
|
||||
extDict: c_int,
|
||||
);
|
||||
fn HUF_getNbBitsFromCTable(ctable: *const usize, symbolValue: u32) -> u32;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ptr_diff(left: *const u8, right: *const u8) -> usize {
|
||||
(left as usize).wrapping_sub(right as usize)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_from(base: *const u8, value: *const u8) -> u32 {
|
||||
ptr_diff(value, base) as u32
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
|
||||
let input_start = input;
|
||||
let word_size = size_of::<usize>();
|
||||
while ptr_diff(input_limit, input) >= word_size {
|
||||
let difference =
|
||||
unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
|
||||
if difference != 0 {
|
||||
let common = if MEM_isLittleEndian() {
|
||||
difference.trailing_zeros()
|
||||
} else {
|
||||
difference.leading_zeros()
|
||||
} as usize
|
||||
/ 8;
|
||||
return ptr_diff(input, input_start) + common;
|
||||
}
|
||||
input = input.wrapping_add(word_size);
|
||||
matched = matched.wrapping_add(word_size);
|
||||
}
|
||||
if MEM_64bits()
|
||||
&& ptr_diff(input_limit, input) >= 4
|
||||
&& unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
|
||||
{
|
||||
input = input.wrapping_add(4);
|
||||
matched = matched.wrapping_add(4);
|
||||
}
|
||||
if ptr_diff(input_limit, input) >= 2
|
||||
&& unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
|
||||
{
|
||||
input = input.wrapping_add(2);
|
||||
matched = matched.wrapping_add(2);
|
||||
}
|
||||
if ptr_diff(input, input_limit) != 0 && unsafe { *matched == *input } {
|
||||
input = input.wrapping_add(1);
|
||||
}
|
||||
ptr_diff(input, input_start)
|
||||
}
|
||||
|
||||
unsafe fn count_2segments(
|
||||
input: *const u8,
|
||||
matched: *const u8,
|
||||
input_end: *const u8,
|
||||
match_end: *const u8,
|
||||
input_start: *const u8,
|
||||
) -> usize {
|
||||
let match_remaining = ptr_diff(match_end, matched);
|
||||
let input_remaining = ptr_diff(input_end, input);
|
||||
let first_end = input.wrapping_add(min(match_remaining, input_remaining));
|
||||
let first_count = unsafe { count(input, matched, first_end) };
|
||||
if matched.wrapping_add(first_count) != match_end {
|
||||
return first_count;
|
||||
}
|
||||
first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn hash_ptr(input: *const u8, hbits: u32, mls: u32) -> usize {
|
||||
let hash_shift32 = |value: u32| {
|
||||
if hbits == 0 {
|
||||
0
|
||||
} else {
|
||||
(value >> (32 - hbits)) as usize
|
||||
}
|
||||
};
|
||||
let hash_shift64 = |value: u64| {
|
||||
if hbits == 0 {
|
||||
0
|
||||
} else {
|
||||
(value >> (64 - hbits)) as usize
|
||||
}
|
||||
};
|
||||
match mls {
|
||||
5 => hash_shift64(
|
||||
unsafe { MEM_readLE64(input.cast::<c_void>()) }
|
||||
.wrapping_shl(24)
|
||||
.wrapping_mul(889_523_592_379),
|
||||
),
|
||||
6 => hash_shift64(
|
||||
unsafe { MEM_readLE64(input.cast::<c_void>()) }
|
||||
.wrapping_shl(16)
|
||||
.wrapping_mul(227_718_039_650_203),
|
||||
),
|
||||
7 => hash_shift64(
|
||||
unsafe { MEM_readLE64(input.cast::<c_void>()) }
|
||||
.wrapping_shl(8)
|
||||
.wrapping_mul(58_295_818_150_454_627),
|
||||
),
|
||||
8 => hash_shift64(
|
||||
unsafe { MEM_readLE64(input.cast::<c_void>()) }.wrapping_mul(0xCF1B_BCDC_B7A5_6463),
|
||||
),
|
||||
_ => hash_shift32(
|
||||
unsafe { MEM_readLE32(input.cast::<c_void>()) }.wrapping_mul(2_654_435_761),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn hash3_ptr(input: *const u8, hbits: u32) -> usize {
|
||||
if hbits == 0 {
|
||||
0
|
||||
} else {
|
||||
let value = unsafe { MEM_readLE32(input.cast::<c_void>()) };
|
||||
(value.wrapping_shl(8).wrapping_mul(PRIME3BYTES) >> (32 - hbits)) as usize
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
|
||||
unsafe { *table.add(index) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
|
||||
unsafe { *table.add(index) = value };
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn lowest_match_index(state: &ZSTD_RustOptState, current: u32) -> u32 {
|
||||
let max_distance = 1u32.wrapping_shl(state.window_log);
|
||||
let within_window = if current.wrapping_sub(state.low_limit) > max_distance {
|
||||
current.wrapping_sub(max_distance)
|
||||
} else {
|
||||
state.low_limit
|
||||
};
|
||||
if state.loaded_dict_end != 0 {
|
||||
state.low_limit
|
||||
} else {
|
||||
within_window
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
|
||||
prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn read_min_match(input: *const u8, length: u32) -> u32 {
|
||||
let value = unsafe { MEM_read32(input.cast::<c_void>()) };
|
||||
if length == 3 && MEM_isLittleEndian() {
|
||||
value << 8
|
||||
} else if length == 3 {
|
||||
value >> 8
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ll_code(value: u32) -> usize {
|
||||
if value > 63 {
|
||||
(ZSTD_highbit32(value) + 19) as usize
|
||||
} else {
|
||||
LL_CODE[value as usize] as usize
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ml_code(value: u32) -> usize {
|
||||
match value {
|
||||
0..=31 => value as usize,
|
||||
32..=33 => 32,
|
||||
34..=35 => 33,
|
||||
36..=37 => 34,
|
||||
38..=39 => 35,
|
||||
40..=43 => 36,
|
||||
44..=47 => 37,
|
||||
48..=51 => 38,
|
||||
52..=55 => 39,
|
||||
56..=71 => 40,
|
||||
72..=87 => 41,
|
||||
88..=127 => 42,
|
||||
_ => (ZSTD_highbit32(value) + 36) as usize,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn bit_weight(stat: u32) -> u32 {
|
||||
ZSTD_highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn frac_weight(raw_stat: u32) -> u32 {
|
||||
let stat = raw_stat.wrapping_add(1);
|
||||
let high_bit = ZSTD_highbit32(stat);
|
||||
let base = high_bit * BITCOST_MULTIPLIER;
|
||||
base + (stat << BITCOST_ACCURACY >> high_bit)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn weight(stat: u32, opt_level: c_int) -> u32 {
|
||||
if opt_level >= 2 {
|
||||
frac_weight(stat)
|
||||
} else {
|
||||
bit_weight(stat)
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn sum_u32(table: *const c_uint, count: usize) -> u32 {
|
||||
let mut total: u32 = 0;
|
||||
for index in 0..count {
|
||||
total = total.wrapping_add(unsafe { *table.add(index) });
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
unsafe fn downscale_stats(
|
||||
table: *mut c_uint,
|
||||
last_index: usize,
|
||||
shift: u32,
|
||||
base_one: bool,
|
||||
) -> u32 {
|
||||
let mut sum: u32 = 0;
|
||||
for index in 0..=last_index {
|
||||
let old = unsafe { *table.add(index) };
|
||||
let value = u32::from(base_one) + (old >> shift);
|
||||
unsafe { *table.add(index) = value };
|
||||
sum = sum.wrapping_add(value);
|
||||
}
|
||||
sum
|
||||
}
|
||||
|
||||
unsafe fn scale_stats(table: *mut c_uint, last_index: usize, log_target: u32) -> u32 {
|
||||
let previous = unsafe { sum_u32(table, last_index + 1) };
|
||||
let factor = previous >> log_target;
|
||||
if factor <= 1 {
|
||||
previous
|
||||
} else {
|
||||
unsafe { downscale_stats(table, last_index, ZSTD_highbit32(factor), true) }
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn fse_max_nb_bits(table: *const u32, symbol: usize) -> u32 {
|
||||
let table_log = unsafe { MEM_read16(table.cast::<c_void>()) } as usize;
|
||||
let transform_offset = 1 + if table_log == 0 {
|
||||
1
|
||||
} else {
|
||||
1 << (table_log - 1)
|
||||
};
|
||||
let transform = unsafe {
|
||||
table
|
||||
.add(transform_offset)
|
||||
.cast::<FseSymbolCompressionTransform>()
|
||||
.add(symbol)
|
||||
.read()
|
||||
};
|
||||
(transform.deltaNbBits.wrapping_add(0xFFFF)) >> 16
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn compressed_literals(opt: &OptState) -> bool {
|
||||
opt.literalCompressionMode != ZSTD_PS_DISABLE
|
||||
}
|
||||
|
||||
unsafe fn set_base_prices(opt: &mut OptState, opt_level: c_int) {
|
||||
if compressed_literals(opt) {
|
||||
opt.litSumBasePrice = weight(opt.litSum, opt_level);
|
||||
}
|
||||
opt.litLengthSumBasePrice = weight(opt.litLengthSum, opt_level);
|
||||
opt.matchLengthSumBasePrice = weight(opt.matchLengthSum, opt_level);
|
||||
opt.offCodeSumBasePrice = weight(opt.offCodeSum, opt_level);
|
||||
}
|
||||
|
||||
unsafe fn rescale_freqs(
|
||||
state: &ZSTD_RustOptState,
|
||||
src: *const u8,
|
||||
src_size: usize,
|
||||
opt_level: c_int,
|
||||
) {
|
||||
let opt = unsafe { &mut *state.opt };
|
||||
let compressed = compressed_literals(opt);
|
||||
opt.priceType = ZOP_DYNAMIC;
|
||||
if opt.litLengthSum == 0 {
|
||||
if src_size <= ZSTD_PREDEF_THRESHOLD {
|
||||
opt.priceType = ZOP_PREDEF;
|
||||
}
|
||||
|
||||
if state.huf_repeat_valid == 0 {
|
||||
/* No valid dictionary table: seed from the first source block. */
|
||||
if compressed {
|
||||
for index in 0..=MAX_LIT {
|
||||
unsafe { *opt.litFreq.add(index) = 0 };
|
||||
}
|
||||
for index in 0..src_size {
|
||||
let byte = unsafe { *src.add(index) } as usize;
|
||||
unsafe { *opt.litFreq.add(byte) = (*opt.litFreq.add(byte)).wrapping_add(1) };
|
||||
}
|
||||
opt.litSum = unsafe { downscale_stats(opt.litFreq, MAX_LIT, 8, false) };
|
||||
}
|
||||
let base_ll: [u32; MAX_LL + 1] = [
|
||||
4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
];
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(base_ll.as_ptr(), opt.litLengthFreq, base_ll.len());
|
||||
}
|
||||
opt.litLengthSum = unsafe { sum_u32(base_ll.as_ptr(), base_ll.len()) };
|
||||
for index in 0..=MAX_ML {
|
||||
unsafe { *opt.matchLengthFreq.add(index) = 1 };
|
||||
}
|
||||
opt.matchLengthSum = (MAX_ML + 1) as u32;
|
||||
let base_of: [u32; MAX_OFF + 1] = [
|
||||
6, 2, 1, 1, 2, 3, 4, 4, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1,
|
||||
];
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(base_of.as_ptr(), opt.offCodeFreq, base_of.len());
|
||||
}
|
||||
opt.offCodeSum = unsafe { sum_u32(base_of.as_ptr(), base_of.len()) };
|
||||
} else {
|
||||
/* Dictionary-seeded tables are projected as leaf pointers by C. */
|
||||
if compressed && !state.huf_ctable.is_null() {
|
||||
opt.litSum = 0;
|
||||
for lit in 0..=MAX_LIT {
|
||||
let bit_cost = unsafe { HUF_getNbBitsFromCTable(state.huf_ctable, lit as u32) };
|
||||
let value = if bit_cost != 0 {
|
||||
1u32 << (11 - bit_cost)
|
||||
} else {
|
||||
1
|
||||
};
|
||||
unsafe { *opt.litFreq.add(lit) = value };
|
||||
opt.litSum = opt.litSum.wrapping_add(value);
|
||||
}
|
||||
}
|
||||
if !state.fse_litlength_ctable.is_null() {
|
||||
opt.litLengthSum = 0;
|
||||
for symbol in 0..=MAX_LL {
|
||||
let bits = unsafe { fse_max_nb_bits(state.fse_litlength_ctable, symbol) };
|
||||
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
|
||||
unsafe { *opt.litLengthFreq.add(symbol) = value };
|
||||
opt.litLengthSum = opt.litLengthSum.wrapping_add(value);
|
||||
}
|
||||
}
|
||||
if !state.fse_matchlength_ctable.is_null() {
|
||||
opt.matchLengthSum = 0;
|
||||
for symbol in 0..=MAX_ML {
|
||||
let bits = unsafe { fse_max_nb_bits(state.fse_matchlength_ctable, symbol) };
|
||||
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
|
||||
unsafe { *opt.matchLengthFreq.add(symbol) = value };
|
||||
opt.matchLengthSum = opt.matchLengthSum.wrapping_add(value);
|
||||
}
|
||||
}
|
||||
if !state.fse_offcode_ctable.is_null() {
|
||||
opt.offCodeSum = 0;
|
||||
for symbol in 0..=MAX_OFF {
|
||||
let bits = unsafe { fse_max_nb_bits(state.fse_offcode_ctable, symbol) };
|
||||
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
|
||||
unsafe { *opt.offCodeFreq.add(symbol) = value };
|
||||
opt.offCodeSum = opt.offCodeSum.wrapping_add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if compressed {
|
||||
opt.litSum = unsafe { scale_stats(opt.litFreq, MAX_LIT, 12) };
|
||||
}
|
||||
opt.litLengthSum = unsafe { scale_stats(opt.litLengthFreq, MAX_LL, 11) };
|
||||
opt.matchLengthSum = unsafe { scale_stats(opt.matchLengthFreq, MAX_ML, 11) };
|
||||
opt.offCodeSum = unsafe { scale_stats(opt.offCodeFreq, MAX_OFF, 11) };
|
||||
}
|
||||
unsafe { set_base_prices(opt, opt_level) };
|
||||
}
|
||||
|
||||
unsafe fn raw_literals_cost(
|
||||
literals: *const u8,
|
||||
lit_length: u32,
|
||||
opt: &OptState,
|
||||
opt_level: c_int,
|
||||
) -> u32 {
|
||||
if lit_length == 0 {
|
||||
return 0;
|
||||
}
|
||||
if !compressed_literals(opt) {
|
||||
return lit_length.wrapping_mul(8).wrapping_mul(BITCOST_MULTIPLIER);
|
||||
}
|
||||
if opt.priceType == ZOP_PREDEF {
|
||||
return lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER);
|
||||
}
|
||||
let mut price = opt.litSumBasePrice.wrapping_mul(lit_length);
|
||||
let max_price = opt.litSumBasePrice.wrapping_sub(BITCOST_MULTIPLIER);
|
||||
for index in 0..lit_length as usize {
|
||||
let literal = unsafe { *literals.add(index) } as usize;
|
||||
let frequency = unsafe { *opt.litFreq.add(literal) };
|
||||
price = price.wrapping_sub(min(weight(frequency, opt_level), max_price));
|
||||
}
|
||||
price
|
||||
}
|
||||
|
||||
unsafe fn lit_length_price(lit_length: u32, opt: &OptState, opt_level: c_int) -> u32 {
|
||||
if opt.priceType == ZOP_PREDEF {
|
||||
return weight(lit_length, opt_level);
|
||||
}
|
||||
if lit_length as usize == ZSTD_BLOCKSIZE_MAX {
|
||||
return BITCOST_MULTIPLIER
|
||||
+ unsafe { lit_length_price((ZSTD_BLOCKSIZE_MAX - 1) as u32, opt, opt_level) };
|
||||
}
|
||||
let code = ll_code(lit_length);
|
||||
(LL_BITS[code] as u32 * BITCOST_MULTIPLIER) + opt.litLengthSumBasePrice
|
||||
- weight(unsafe { *opt.litLengthFreq.add(code) }, opt_level)
|
||||
}
|
||||
|
||||
unsafe fn get_match_price(
|
||||
off_base: u32,
|
||||
match_length: u32,
|
||||
opt: &OptState,
|
||||
opt_level: c_int,
|
||||
) -> u32 {
|
||||
let off_code = ZSTD_highbit32(off_base);
|
||||
let ml_base = match_length - MINMATCH as u32;
|
||||
if opt.priceType == ZOP_PREDEF {
|
||||
return weight(ml_base, opt_level) + (16u32 + off_code).wrapping_mul(BITCOST_MULTIPLIER);
|
||||
}
|
||||
let mut price = off_code * BITCOST_MULTIPLIER + opt.offCodeSumBasePrice
|
||||
- weight(
|
||||
unsafe { *opt.offCodeFreq.add(off_code as usize) },
|
||||
opt_level,
|
||||
);
|
||||
if opt_level < 2 && off_code >= 20 {
|
||||
price += (off_code - 19) * 2 * BITCOST_MULTIPLIER;
|
||||
}
|
||||
let code = ml_code(ml_base);
|
||||
price += ML_BITS[code] as u32 * BITCOST_MULTIPLIER + opt.matchLengthSumBasePrice
|
||||
- weight(unsafe { *opt.matchLengthFreq.add(code) }, opt_level);
|
||||
price + BITCOST_MULTIPLIER / 5
|
||||
}
|
||||
|
||||
unsafe fn update_stats(
|
||||
opt: &mut OptState,
|
||||
lit_length: u32,
|
||||
literals: *const u8,
|
||||
off_base: u32,
|
||||
match_length: u32,
|
||||
) {
|
||||
if compressed_literals(opt) {
|
||||
for index in 0..lit_length as usize {
|
||||
let literal = unsafe { *literals.add(index) } as usize;
|
||||
let frequency = unsafe { opt.litFreq.add(literal) };
|
||||
unsafe { *frequency = (*frequency).wrapping_add(ZSTD_LITFREQ_ADD) };
|
||||
}
|
||||
opt.litSum = opt
|
||||
.litSum
|
||||
.wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD));
|
||||
}
|
||||
let ll = ll_code(lit_length);
|
||||
unsafe { *opt.litLengthFreq.add(ll) = (*opt.litLengthFreq.add(ll)).wrapping_add(1) };
|
||||
opt.litLengthSum = opt.litLengthSum.wrapping_add(1);
|
||||
let off = ZSTD_highbit32(off_base) as usize;
|
||||
unsafe { *opt.offCodeFreq.add(off) = (*opt.offCodeFreq.add(off)).wrapping_add(1) };
|
||||
opt.offCodeSum = opt.offCodeSum.wrapping_add(1);
|
||||
let ml = ml_code(match_length - MINMATCH as u32);
|
||||
unsafe { *opt.matchLengthFreq.add(ml) = (*opt.matchLengthFreq.add(ml)).wrapping_add(1) };
|
||||
opt.matchLengthSum = opt.matchLengthSum.wrapping_add(1);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn offset_to_offbase(offset: u32) -> u32 {
|
||||
offset.wrapping_add(OFFSET_OFFBASE)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn repcode_to_offbase(rep_code: u32) -> u32 {
|
||||
rep_code
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn new_rep(mut rep: [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] {
|
||||
if off_base > OFFSET_OFFBASE {
|
||||
rep[2] = rep[1];
|
||||
rep[1] = rep[0];
|
||||
rep[0] = off_base - OFFSET_OFFBASE;
|
||||
} else {
|
||||
let rep_code = off_base - 1 + u32::from(ll0);
|
||||
if rep_code != 0 {
|
||||
let current = if rep_code == ZSTD_REP_NUM as u32 {
|
||||
rep[0].wrapping_sub(1)
|
||||
} else {
|
||||
rep[rep_code as usize]
|
||||
};
|
||||
if rep_code >= 2 {
|
||||
rep[2] = rep[1];
|
||||
}
|
||||
rep[1] = rep[0];
|
||||
rep[0] = current;
|
||||
}
|
||||
}
|
||||
rep
|
||||
}
|
||||
|
||||
unsafe fn store_seq(
|
||||
seq_store: *mut SeqStore_t,
|
||||
lit_length: usize,
|
||||
literals: *const u8,
|
||||
off_base: u32,
|
||||
match_length: usize,
|
||||
) {
|
||||
let store = unsafe { &mut *seq_store };
|
||||
let sequence = store.sequences;
|
||||
let index =
|
||||
ptr_diff(sequence.cast::<u8>(), store.sequencesStart.cast::<u8>()) / size_of::<SeqDef>();
|
||||
if lit_length != 0 {
|
||||
unsafe { ptr::copy_nonoverlapping(literals, store.lit, lit_length) };
|
||||
}
|
||||
store.lit = store.lit.wrapping_add(lit_length);
|
||||
if lit_length > u16::MAX as usize {
|
||||
store.longLengthType = 1;
|
||||
store.longLengthPos = index as u32;
|
||||
}
|
||||
unsafe {
|
||||
(*sequence).litLength = lit_length as u16;
|
||||
(*sequence).offBase = off_base;
|
||||
}
|
||||
let ml_base = match_length - MINMATCH;
|
||||
if ml_base > u16::MAX as usize {
|
||||
store.longLengthType = 2;
|
||||
store.longLengthPos = index as u32;
|
||||
}
|
||||
unsafe { (*sequence).mlBase = ml_base as u16 };
|
||||
store.sequences = store.sequences.wrapping_add(1);
|
||||
}
|
||||
|
||||
unsafe fn skip_raw_seq_store_bytes(store: &mut RawSeqStore, nb_bytes: usize) {
|
||||
let mut current = (store.posInSequence + nb_bytes) as u32;
|
||||
while current != 0 && store.pos < store.size {
|
||||
let sequence = unsafe { store.seq.add(store.pos).read() };
|
||||
let sequence_size = sequence.litLength.wrapping_add(sequence.matchLength);
|
||||
if current >= sequence_size {
|
||||
current -= sequence_size;
|
||||
store.pos += 1;
|
||||
} else {
|
||||
store.posInSequence = current as usize;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if current == 0 || store.pos == store.size {
|
||||
store.posInSequence = 0;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct OptLdm {
|
||||
seq_store: RawSeqStore,
|
||||
start_pos: u32,
|
||||
end_pos: u32,
|
||||
offset: u32,
|
||||
}
|
||||
|
||||
unsafe fn ldm_next_match(opt_ldm: &mut OptLdm, current_pos: u32, block_remaining: u32) {
|
||||
if opt_ldm.seq_store.size == 0 || opt_ldm.seq_store.pos >= opt_ldm.seq_store.size {
|
||||
opt_ldm.start_pos = u32::MAX;
|
||||
opt_ldm.end_pos = u32::MAX;
|
||||
return;
|
||||
}
|
||||
let sequence = unsafe { opt_ldm.seq_store.seq.add(opt_ldm.seq_store.pos).read() };
|
||||
let block_end = current_pos.wrapping_add(block_remaining);
|
||||
let literals_remaining =
|
||||
(sequence.litLength as usize).saturating_sub(opt_ldm.seq_store.posInSequence) as u32;
|
||||
let match_remaining = if literals_remaining == 0 {
|
||||
sequence
|
||||
.matchLength
|
||||
.wrapping_sub(opt_ldm.seq_store.posInSequence as u32 - sequence.litLength)
|
||||
} else {
|
||||
sequence.matchLength
|
||||
};
|
||||
if literals_remaining >= block_remaining {
|
||||
opt_ldm.start_pos = u32::MAX;
|
||||
opt_ldm.end_pos = u32::MAX;
|
||||
unsafe { skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, block_remaining as usize) };
|
||||
return;
|
||||
}
|
||||
opt_ldm.start_pos = current_pos.wrapping_add(literals_remaining);
|
||||
opt_ldm.end_pos = opt_ldm.start_pos.wrapping_add(match_remaining);
|
||||
opt_ldm.offset = sequence.offset;
|
||||
if opt_ldm.end_pos > block_end {
|
||||
opt_ldm.end_pos = block_end;
|
||||
unsafe {
|
||||
skip_raw_seq_store_bytes(
|
||||
&mut opt_ldm.seq_store,
|
||||
block_end.wrapping_sub(current_pos) as usize,
|
||||
)
|
||||
};
|
||||
} else {
|
||||
unsafe {
|
||||
skip_raw_seq_store_bytes(
|
||||
&mut opt_ldm.seq_store,
|
||||
(literals_remaining + match_remaining) as usize,
|
||||
)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn ldm_maybe_add_match(
|
||||
matches: *mut ZSTD_match_t,
|
||||
nb_matches: &mut u32,
|
||||
opt_ldm: &OptLdm,
|
||||
current_pos: u32,
|
||||
min_match: u32,
|
||||
) {
|
||||
let position_diff = current_pos.wrapping_sub(opt_ldm.start_pos);
|
||||
let candidate_length = opt_ldm
|
||||
.end_pos
|
||||
.wrapping_sub(opt_ldm.start_pos)
|
||||
.wrapping_sub(position_diff);
|
||||
if current_pos < opt_ldm.start_pos
|
||||
|| current_pos >= opt_ldm.end_pos
|
||||
|| candidate_length < min_match
|
||||
{
|
||||
return;
|
||||
}
|
||||
if *nb_matches == 0
|
||||
|| (candidate_length > unsafe { (*matches.add(*nb_matches as usize - 1)).len }
|
||||
&& *nb_matches < ZSTD_OPT_NUM)
|
||||
{
|
||||
let slot = unsafe { &mut *matches.add(*nb_matches as usize) };
|
||||
slot.len = candidate_length;
|
||||
slot.off = offset_to_offbase(opt_ldm.offset);
|
||||
*nb_matches += 1;
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn ldm_process_match(
|
||||
opt_ldm: &mut OptLdm,
|
||||
matches: *mut ZSTD_match_t,
|
||||
nb_matches: &mut u32,
|
||||
current_pos: u32,
|
||||
remaining: u32,
|
||||
min_match: u32,
|
||||
) {
|
||||
if opt_ldm.seq_store.size == 0 || opt_ldm.seq_store.pos >= opt_ldm.seq_store.size {
|
||||
return;
|
||||
}
|
||||
if current_pos >= opt_ldm.end_pos {
|
||||
if current_pos > opt_ldm.end_pos {
|
||||
unsafe {
|
||||
skip_raw_seq_store_bytes(
|
||||
&mut opt_ldm.seq_store,
|
||||
current_pos.wrapping_sub(opt_ldm.end_pos) as usize,
|
||||
)
|
||||
};
|
||||
}
|
||||
unsafe { ldm_next_match(opt_ldm, current_pos, remaining) };
|
||||
}
|
||||
unsafe { ldm_maybe_add_match(matches, nb_matches, opt_ldm, current_pos, min_match) };
|
||||
}
|
||||
|
||||
unsafe fn insert_and_find_first_index_hash3(
|
||||
state: &ZSTD_RustOptState,
|
||||
next_to_update3: &mut u32,
|
||||
ip: *const u8,
|
||||
) -> u32 {
|
||||
let target = index_from(state.base, ip);
|
||||
let hash = unsafe { hash3_ptr(ip, state.hash_log3) };
|
||||
let mut index = *next_to_update3;
|
||||
while index < target {
|
||||
let position = state.base.wrapping_add(index as usize);
|
||||
let position_hash = unsafe { hash3_ptr(position, state.hash_log3) };
|
||||
unsafe { table_set(state.hash_table3, position_hash, index) };
|
||||
index = index.wrapping_add(1);
|
||||
}
|
||||
*next_to_update3 = target;
|
||||
unsafe { table_get(state.hash_table3, hash) }
|
||||
}
|
||||
|
||||
unsafe fn insert_bt_and_get_all_matches(
|
||||
state: &ZSTD_RustOptState,
|
||||
next_to_update3: &mut u32,
|
||||
ip: *const u8,
|
||||
input_limit: *const u8,
|
||||
dict_mode: c_int,
|
||||
rep: &[u32; ZSTD_REP_NUM],
|
||||
ll0: u32,
|
||||
length_to_beat: u32,
|
||||
mls: u32,
|
||||
) -> u32 {
|
||||
let sufficient_len = min(state.target_length, ZSTD_OPT_NUM - 1);
|
||||
let current = index_from(state.base, ip);
|
||||
let min_match = if mls == 3 { 3 } else { 4 };
|
||||
let hash = unsafe { hash_ptr(ip, state.hash_log, mls) };
|
||||
let mut match_index = unsafe { table_get(state.hash_table, hash) };
|
||||
let bt_log = state.chain_log.wrapping_sub(1);
|
||||
let bt_mask = (1u32 << bt_log).wrapping_sub(1);
|
||||
let dict_limit = state.dict_limit;
|
||||
let dict_end = state.dict_base.wrapping_add(dict_limit as usize);
|
||||
let prefix_start = state.base.wrapping_add(dict_limit as usize);
|
||||
let bt_low = current.saturating_sub(bt_mask);
|
||||
let window_low = lowest_match_index(state, current);
|
||||
let match_low = if window_low == 0 { 1 } else { window_low };
|
||||
let mut smaller = unsafe { state.chain_table.add(2 * (current & bt_mask) as usize) };
|
||||
let mut larger = unsafe { smaller.add(1) };
|
||||
let mut dummy = 0u32;
|
||||
let mut match_end_index = current.wrapping_add(HASH_READ_SIZE as u32 + 1);
|
||||
let mut common_smaller = 0usize;
|
||||
let mut common_larger = 0usize;
|
||||
let mut best_length = length_to_beat.wrapping_sub(1) as usize;
|
||||
let mut match_count = 0u32;
|
||||
let mut nb_compares = 1u32 << state.search_log;
|
||||
|
||||
let dms = if dict_mode == DICT_MATCH_STATE {
|
||||
debug_assert!(!state.dict_match_state.is_null());
|
||||
unsafe { &*state.dict_match_state }
|
||||
} else {
|
||||
state
|
||||
};
|
||||
let dms_base = dms.base;
|
||||
let dms_end = dms.next_src;
|
||||
let dms_high_limit = if dict_mode == DICT_MATCH_STATE {
|
||||
index_from(dms_base, dms_end)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let dms_low_limit = if dict_mode == DICT_MATCH_STATE {
|
||||
dms.low_limit
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let dms_index_delta = if dict_mode == DICT_MATCH_STATE {
|
||||
window_low.wrapping_sub(dms_high_limit)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let dms_hash_log = if dict_mode == DICT_MATCH_STATE {
|
||||
dms.hash_log
|
||||
} else {
|
||||
state.hash_log
|
||||
};
|
||||
let dms_bt_log = if dict_mode == DICT_MATCH_STATE {
|
||||
dms.chain_log.wrapping_sub(1)
|
||||
} else {
|
||||
bt_log
|
||||
};
|
||||
let dms_bt_mask = if dict_mode == DICT_MATCH_STATE {
|
||||
(1u32 << dms_bt_log).wrapping_sub(1)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let dms_bt_low = if dict_mode == DICT_MATCH_STATE
|
||||
&& dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit)
|
||||
{
|
||||
dms_high_limit.wrapping_sub(dms_bt_mask)
|
||||
} else {
|
||||
dms_low_limit
|
||||
};
|
||||
|
||||
for rep_code in ll0..(ZSTD_REP_NUM as u32 + ll0) {
|
||||
let rep_offset = if rep_code == ZSTD_REP_NUM as u32 {
|
||||
rep[0].wrapping_sub(1)
|
||||
} else {
|
||||
rep[rep_code as usize]
|
||||
};
|
||||
let rep_index = current.wrapping_sub(rep_offset);
|
||||
let mut rep_len = 0usize;
|
||||
if rep_offset.wrapping_sub(1) < current.wrapping_sub(dict_limit) {
|
||||
if rep_index >= window_low
|
||||
&& unsafe { read_min_match(ip, min_match) }
|
||||
== unsafe { read_min_match(ip.wrapping_sub(rep_offset as usize), min_match) }
|
||||
{
|
||||
rep_len = unsafe {
|
||||
count(
|
||||
ip.wrapping_add(min_match as usize),
|
||||
ip.wrapping_sub(rep_offset as usize)
|
||||
.wrapping_add(min_match as usize),
|
||||
input_limit,
|
||||
) + min_match as usize
|
||||
};
|
||||
}
|
||||
} else {
|
||||
let rep_match = if dict_mode == DICT_MATCH_STATE {
|
||||
dms_base.wrapping_add(rep_index.wrapping_sub(dms_index_delta) as usize)
|
||||
} else {
|
||||
state.dict_base.wrapping_add(rep_index as usize)
|
||||
};
|
||||
if dict_mode == DICT_EXT
|
||||
&& rep_offset.wrapping_sub(1) < current.wrapping_sub(window_low)
|
||||
&& index_overlap_check(dict_limit, rep_index)
|
||||
&& unsafe { read_min_match(ip, min_match) }
|
||||
== unsafe { read_min_match(rep_match, min_match) }
|
||||
{
|
||||
rep_len = unsafe {
|
||||
count_2segments(
|
||||
ip.wrapping_add(min_match as usize),
|
||||
rep_match.wrapping_add(min_match as usize),
|
||||
input_limit,
|
||||
dict_end,
|
||||
prefix_start,
|
||||
) + min_match as usize
|
||||
};
|
||||
}
|
||||
if dict_mode == DICT_MATCH_STATE
|
||||
&& rep_offset.wrapping_sub(1)
|
||||
< current.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta))
|
||||
&& index_overlap_check(dict_limit, rep_index)
|
||||
&& unsafe { read_min_match(ip, min_match) }
|
||||
== unsafe { read_min_match(rep_match, min_match) }
|
||||
{
|
||||
rep_len = unsafe {
|
||||
count_2segments(
|
||||
ip.wrapping_add(min_match as usize),
|
||||
rep_match.wrapping_add(min_match as usize),
|
||||
input_limit,
|
||||
dms_end,
|
||||
prefix_start,
|
||||
) + min_match as usize
|
||||
};
|
||||
}
|
||||
}
|
||||
if rep_len > best_length {
|
||||
best_length = rep_len;
|
||||
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
|
||||
slot.off = repcode_to_offbase(rep_code - ll0 + 1);
|
||||
slot.len = rep_len as u32;
|
||||
match_count += 1;
|
||||
if rep_len > sufficient_len as usize || ip.wrapping_add(rep_len) == input_limit {
|
||||
return match_count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if mls == 3 && best_length < mls as usize {
|
||||
let match_index3 = unsafe { insert_and_find_first_index_hash3(state, next_to_update3, ip) };
|
||||
if match_index3 >= match_low && current.wrapping_sub(match_index3) < (1 << 18) {
|
||||
let match_ptr = if dict_mode == DICT_EXT && match_index3 < dict_limit {
|
||||
state.dict_base.wrapping_add(match_index3 as usize)
|
||||
} else {
|
||||
state.base.wrapping_add(match_index3 as usize)
|
||||
};
|
||||
let match_length = if dict_mode == DICT_EXT && match_index3 < dict_limit {
|
||||
unsafe { count_2segments(ip, match_ptr, input_limit, dict_end, prefix_start) }
|
||||
} else {
|
||||
unsafe { count(ip, match_ptr, input_limit) }
|
||||
};
|
||||
if match_length >= mls as usize {
|
||||
let slot = unsafe { &mut *state.opt_match_table() };
|
||||
slot.off = offset_to_offbase(current.wrapping_sub(match_index3));
|
||||
slot.len = match_length as u32;
|
||||
if match_length > sufficient_len as usize
|
||||
|| ip.wrapping_add(match_length) == input_limit
|
||||
{
|
||||
unsafe { *state.next_to_update = current.wrapping_add(1) };
|
||||
return 1;
|
||||
}
|
||||
match_count = 1;
|
||||
best_length = match_length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe { table_set(state.hash_table, hash, current) };
|
||||
while nb_compares != 0 && match_index >= match_low {
|
||||
nb_compares -= 1;
|
||||
let next_ptr = unsafe { state.chain_table.add(2 * (match_index & bt_mask) as usize) };
|
||||
let mut match_length = min(common_smaller, common_larger);
|
||||
let mut match_ptr: *const u8;
|
||||
if dict_mode != DICT_EXT || match_index.wrapping_add(match_length as u32) >= dict_limit {
|
||||
match_ptr = state.base.wrapping_add(match_index as usize);
|
||||
match_length += unsafe {
|
||||
count(
|
||||
ip.wrapping_add(match_length),
|
||||
match_ptr.wrapping_add(match_length),
|
||||
input_limit,
|
||||
)
|
||||
};
|
||||
} else {
|
||||
match_ptr = state.dict_base.wrapping_add(match_index as usize);
|
||||
match_length += unsafe {
|
||||
count_2segments(
|
||||
ip.wrapping_add(match_length),
|
||||
match_ptr.wrapping_add(match_length),
|
||||
input_limit,
|
||||
dict_end,
|
||||
prefix_start,
|
||||
)
|
||||
};
|
||||
if match_index.wrapping_add(match_length as u32) >= dict_limit {
|
||||
match_ptr = state.base.wrapping_add(match_index as usize);
|
||||
}
|
||||
}
|
||||
|
||||
if match_length > best_length {
|
||||
if match_length > match_end_index.wrapping_sub(match_index) as usize {
|
||||
match_end_index = match_index.wrapping_add(match_length as u32);
|
||||
}
|
||||
best_length = match_length;
|
||||
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
|
||||
slot.off = offset_to_offbase(current.wrapping_sub(match_index));
|
||||
slot.len = match_length as u32;
|
||||
match_count += 1;
|
||||
if match_length > ZSTD_OPT_NUM as usize || ip.wrapping_add(match_length) == input_limit
|
||||
{
|
||||
if dict_mode == DICT_MATCH_STATE {
|
||||
nb_compares = 0;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let next_byte = unsafe { *match_ptr.add(match_length) };
|
||||
let input_byte = unsafe { *ip.add(match_length) };
|
||||
if next_byte < input_byte {
|
||||
unsafe { *smaller = match_index };
|
||||
common_smaller = match_length;
|
||||
if match_index <= bt_low {
|
||||
smaller = &mut dummy;
|
||||
break;
|
||||
}
|
||||
smaller = unsafe { next_ptr.add(1) };
|
||||
match_index = unsafe { *next_ptr.add(1) };
|
||||
} else {
|
||||
unsafe { *larger = match_index };
|
||||
common_larger = match_length;
|
||||
if match_index <= bt_low {
|
||||
larger = &mut dummy;
|
||||
break;
|
||||
}
|
||||
larger = next_ptr;
|
||||
match_index = unsafe { *next_ptr };
|
||||
}
|
||||
}
|
||||
unsafe {
|
||||
*smaller = 0;
|
||||
*larger = 0;
|
||||
}
|
||||
|
||||
if dict_mode == DICT_MATCH_STATE && nb_compares != 0 {
|
||||
let dict_hash = unsafe { hash_ptr(ip, dms_hash_log, mls) };
|
||||
let mut dict_match_index = unsafe { table_get(dms.hash_table, dict_hash) };
|
||||
common_smaller = 0;
|
||||
common_larger = 0;
|
||||
while nb_compares != 0 && dict_match_index > dms_low_limit {
|
||||
nb_compares -= 1;
|
||||
let next_ptr = unsafe {
|
||||
dms.chain_table
|
||||
.add(2 * (dict_match_index & dms_bt_mask) as usize)
|
||||
};
|
||||
let mut match_length = min(common_smaller, common_larger);
|
||||
let mut match_ptr = dms_base.wrapping_add(dict_match_index as usize);
|
||||
match_length += unsafe {
|
||||
count_2segments(
|
||||
ip.wrapping_add(match_length),
|
||||
match_ptr.wrapping_add(match_length),
|
||||
input_limit,
|
||||
dms_end,
|
||||
prefix_start,
|
||||
)
|
||||
};
|
||||
if dict_match_index.wrapping_add(match_length as u32) >= dms_high_limit {
|
||||
match_ptr = state
|
||||
.base
|
||||
.wrapping_add(dict_match_index.wrapping_add(dms_index_delta) as usize);
|
||||
}
|
||||
if match_length > best_length {
|
||||
let prefix_index = dict_match_index.wrapping_add(dms_index_delta);
|
||||
if match_length > match_end_index.wrapping_sub(prefix_index) as usize {
|
||||
match_end_index = prefix_index.wrapping_add(match_length as u32);
|
||||
}
|
||||
best_length = match_length;
|
||||
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
|
||||
slot.off = offset_to_offbase(current.wrapping_sub(prefix_index));
|
||||
slot.len = match_length as u32;
|
||||
match_count += 1;
|
||||
if match_length > ZSTD_OPT_NUM as usize
|
||||
|| ip.wrapping_add(match_length) == input_limit
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
if dict_match_index <= dms_bt_low {
|
||||
break;
|
||||
}
|
||||
if unsafe { *match_ptr.add(match_length) } < unsafe { *ip.add(match_length) } {
|
||||
common_smaller = match_length;
|
||||
dict_match_index = unsafe { *next_ptr.add(1) };
|
||||
} else {
|
||||
common_larger = match_length;
|
||||
dict_match_index = unsafe { *next_ptr };
|
||||
}
|
||||
}
|
||||
}
|
||||
unsafe { *state.next_to_update = match_end_index.wrapping_sub(8) };
|
||||
match_count
|
||||
}
|
||||
|
||||
impl ZSTD_RustOptState {
|
||||
#[inline]
|
||||
unsafe fn opt_match_table(&self) -> *mut ZSTD_match_t {
|
||||
unsafe { (*self.opt).matchTable }
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn get_all_matches(
|
||||
state: &mut ZSTD_RustOptState,
|
||||
next_to_update3: &mut u32,
|
||||
ip: *const u8,
|
||||
input_limit: *const u8,
|
||||
rep: &[u32; ZSTD_REP_NUM],
|
||||
ll0: u32,
|
||||
length_to_beat: u32,
|
||||
dict_mode: c_int,
|
||||
mls: u32,
|
||||
) -> u32 {
|
||||
debug_assert!((3..=6).contains(&mls));
|
||||
if (ip as usize)
|
||||
< state
|
||||
.base
|
||||
.wrapping_add(unsafe { *state.next_to_update } as usize) as usize
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
unsafe {
|
||||
ZSTD_rust_opt_updateTreeInternal(
|
||||
(state as *mut ZSTD_RustOptState).cast::<OptTreePrefix>(),
|
||||
ip.cast::<c_void>(),
|
||||
input_limit.cast::<c_void>(),
|
||||
mls,
|
||||
c_int::from(dict_mode == DICT_EXT),
|
||||
);
|
||||
insert_bt_and_get_all_matches(
|
||||
state,
|
||||
next_to_update3,
|
||||
ip,
|
||||
input_limit,
|
||||
dict_mode,
|
||||
rep,
|
||||
ll0,
|
||||
length_to_beat,
|
||||
mls,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn resolve_shortest_path(
|
||||
state: &mut ZSTD_RustOptState,
|
||||
seq_store: *mut SeqStore_t,
|
||||
rep: &mut [u32; ZSTD_REP_NUM],
|
||||
istart: *const u8,
|
||||
iend: *const u8,
|
||||
anchor: &mut *const u8,
|
||||
ip: &mut *const u8,
|
||||
mut cur: u32,
|
||||
last_pos: u32,
|
||||
last_stretch: ZSTD_optimal_t,
|
||||
opt_level: c_int,
|
||||
) {
|
||||
let opt_state = unsafe { &mut *state.opt };
|
||||
let opt = opt_state.priceTable;
|
||||
if last_stretch.mlen == 0 {
|
||||
*ip = ip.wrapping_add(last_pos as usize);
|
||||
return;
|
||||
}
|
||||
|
||||
if last_stretch.litlen == 0 {
|
||||
let previous = unsafe { (*opt.add(cur as usize)).rep };
|
||||
*rep = new_rep(previous, last_stretch.off, unsafe {
|
||||
(*opt.add(cur as usize)).litlen == 0
|
||||
});
|
||||
} else {
|
||||
*rep = last_stretch.rep;
|
||||
cur = cur.wrapping_sub(last_stretch.litlen);
|
||||
}
|
||||
|
||||
let store_end = cur + 2;
|
||||
let mut store_start = store_end;
|
||||
let mut stretch_pos = cur;
|
||||
unsafe {
|
||||
*opt.add(store_end as usize) = last_stretch;
|
||||
}
|
||||
|
||||
loop {
|
||||
let next_stretch = unsafe { *opt.add(stretch_pos as usize) };
|
||||
unsafe { (*opt.add(store_start as usize)).litlen = next_stretch.litlen };
|
||||
if next_stretch.mlen == 0 {
|
||||
break;
|
||||
}
|
||||
store_start -= 1;
|
||||
unsafe { *opt.add(store_start as usize) = next_stretch };
|
||||
stretch_pos = stretch_pos.wrapping_sub(next_stretch.litlen + next_stretch.mlen);
|
||||
}
|
||||
|
||||
let mut store_pos = store_start;
|
||||
while store_pos <= store_end {
|
||||
let current = unsafe { *opt.add(store_pos as usize) };
|
||||
let lit_length = current.litlen;
|
||||
let match_length = current.mlen;
|
||||
if match_length == 0 {
|
||||
*ip = anchor.wrapping_add(lit_length as usize);
|
||||
break;
|
||||
}
|
||||
unsafe {
|
||||
update_stats(opt_state, lit_length, *anchor, current.off, match_length);
|
||||
store_seq(
|
||||
seq_store,
|
||||
lit_length as usize,
|
||||
*anchor,
|
||||
current.off,
|
||||
match_length as usize,
|
||||
);
|
||||
}
|
||||
*anchor = anchor.wrapping_add((lit_length + match_length) as usize);
|
||||
*ip = *anchor;
|
||||
store_pos += 1;
|
||||
}
|
||||
unsafe { set_base_prices(opt_state, opt_level) };
|
||||
let _ = istart;
|
||||
let _ = iend;
|
||||
}
|
||||
|
||||
unsafe fn compress_block_opt_generic(
|
||||
state: &mut ZSTD_RustOptState,
|
||||
seq_store: *mut SeqStore_t,
|
||||
reps: *mut u32,
|
||||
src: *const u8,
|
||||
src_size: usize,
|
||||
opt_level: c_int,
|
||||
dict_mode: c_int,
|
||||
) -> usize {
|
||||
let istart = src;
|
||||
let mut ip = istart;
|
||||
let mut anchor = istart;
|
||||
let iend = src.wrapping_add(src_size);
|
||||
let ilimit = if src_size >= HASH_READ_SIZE {
|
||||
iend.wrapping_sub(HASH_READ_SIZE)
|
||||
} else {
|
||||
istart
|
||||
};
|
||||
let prefix_start = state.base.wrapping_add(state.dict_limit as usize);
|
||||
let sufficient_len = min(state.target_length, ZSTD_OPT_NUM - 1);
|
||||
let min_match = if state.min_match == 3 { 3 } else { 4 };
|
||||
let mut next_to_update3 = unsafe { *state.next_to_update };
|
||||
let opt_state = unsafe { &mut *state.opt };
|
||||
let matches = opt_state.matchTable;
|
||||
let opt = opt_state.priceTable;
|
||||
let mut last_stretch = ZSTD_optimal_t::default();
|
||||
let mut opt_ldm = OptLdm {
|
||||
seq_store: if state.ldm_seq_store.is_null() {
|
||||
RawSeqStore::default()
|
||||
} else {
|
||||
unsafe { *state.ldm_seq_store }
|
||||
},
|
||||
start_pos: 0,
|
||||
end_pos: 0,
|
||||
offset: 0,
|
||||
};
|
||||
|
||||
unsafe { ldm_next_match(&mut opt_ldm, 0, src_size as u32) };
|
||||
unsafe { rescale_freqs(state, src, src_size, opt_level) };
|
||||
if ip == prefix_start {
|
||||
ip = ip.wrapping_add(1);
|
||||
}
|
||||
|
||||
while (ip as usize) < ilimit as usize {
|
||||
let mut last_pos;
|
||||
let mut found_immediate = false;
|
||||
let litlen = ptr_diff(ip, anchor) as u32;
|
||||
let ll0 = u32::from(litlen == 0);
|
||||
let mut nb_matches = unsafe {
|
||||
get_all_matches(
|
||||
state,
|
||||
&mut next_to_update3,
|
||||
ip,
|
||||
iend,
|
||||
&[*reps, *reps.add(1), *reps.add(2)],
|
||||
ll0,
|
||||
min_match,
|
||||
dict_mode,
|
||||
min_match,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
ldm_process_match(
|
||||
&mut opt_ldm,
|
||||
matches,
|
||||
&mut nb_matches,
|
||||
ptr_diff(ip, istart) as u32,
|
||||
ptr_diff(iend, ip) as u32,
|
||||
min_match,
|
||||
)
|
||||
};
|
||||
if nb_matches == 0 {
|
||||
ip = ip.wrapping_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
unsafe {
|
||||
(*opt).mlen = 0;
|
||||
(*opt).litlen = litlen;
|
||||
(*opt).price = lit_length_price(litlen, opt_state, opt_level) as c_int;
|
||||
(*opt).rep = [*reps, *reps.add(1), *reps.add(2)];
|
||||
}
|
||||
|
||||
let max_match = unsafe { *matches.add(nb_matches as usize - 1) };
|
||||
if max_match.len > sufficient_len {
|
||||
last_stretch.litlen = 0;
|
||||
last_stretch.mlen = max_match.len;
|
||||
last_stretch.off = max_match.off;
|
||||
last_pos = max_match.len;
|
||||
found_immediate = true;
|
||||
} else {
|
||||
let mut pos = 1u32;
|
||||
while pos < min_match {
|
||||
unsafe {
|
||||
(*opt.add(pos as usize)).price = ZSTD_MAX_PRICE;
|
||||
(*opt.add(pos as usize)).mlen = 0;
|
||||
(*opt.add(pos as usize)).litlen = litlen + pos;
|
||||
}
|
||||
pos += 1;
|
||||
}
|
||||
for match_index in 0..nb_matches as usize {
|
||||
let candidate = unsafe { *matches.add(match_index) };
|
||||
while pos <= candidate.len {
|
||||
let match_price =
|
||||
unsafe { get_match_price(candidate.off, pos, opt_state, opt_level) }
|
||||
as c_int;
|
||||
let sequence_price = unsafe { (*opt).price } + match_price;
|
||||
unsafe {
|
||||
(*opt.add(pos as usize)).mlen = pos;
|
||||
(*opt.add(pos as usize)).off = candidate.off;
|
||||
(*opt.add(pos as usize)).litlen = 0;
|
||||
(*opt.add(pos as usize)).price =
|
||||
sequence_price + lit_length_price(0, opt_state, opt_level) as c_int;
|
||||
}
|
||||
pos += 1;
|
||||
}
|
||||
}
|
||||
last_pos = pos - 1;
|
||||
unsafe { (*opt.add(pos as usize)).price = ZSTD_MAX_PRICE };
|
||||
}
|
||||
|
||||
if found_immediate {
|
||||
let mut next_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
|
||||
unsafe {
|
||||
resolve_shortest_path(
|
||||
state,
|
||||
seq_store,
|
||||
&mut next_rep,
|
||||
istart,
|
||||
iend,
|
||||
&mut anchor,
|
||||
&mut ip,
|
||||
0,
|
||||
last_pos,
|
||||
last_stretch,
|
||||
opt_level,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
*reps = next_rep[0];
|
||||
*reps.add(1) = next_rep[1];
|
||||
*reps.add(2) = next_rep[2];
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut cur = 1u32;
|
||||
let mut resolved_early = false;
|
||||
while cur <= last_pos {
|
||||
let inr = ip.wrapping_add(cur as usize);
|
||||
let literal_length = unsafe { (*opt.add((cur - 1) as usize)).litlen } + 1;
|
||||
let price = unsafe { (*opt.add((cur - 1) as usize)).price }
|
||||
+ unsafe {
|
||||
raw_literals_cost(ip.wrapping_add((cur - 1) as usize), 1, opt_state, opt_level)
|
||||
} as c_int
|
||||
+ (unsafe { lit_length_price(literal_length, opt_state, opt_level) } as c_int
|
||||
- unsafe { lit_length_price(literal_length - 1, opt_state, opt_level) }
|
||||
as c_int);
|
||||
if price <= unsafe { (*opt.add(cur as usize)).price } {
|
||||
let previous_match = unsafe { *opt.add(cur as usize) };
|
||||
let previous = unsafe { *opt.add((cur - 1) as usize) };
|
||||
unsafe { *opt.add(cur as usize) = previous };
|
||||
unsafe {
|
||||
(*opt.add(cur as usize)).litlen = literal_length;
|
||||
(*opt.add(cur as usize)).price = price;
|
||||
}
|
||||
if opt_level >= 1
|
||||
&& previous_match.litlen == 0
|
||||
&& (unsafe { lit_length_price(1, opt_state, opt_level) } as c_int
|
||||
- unsafe { lit_length_price(0, opt_state, opt_level) } as c_int)
|
||||
< 0
|
||||
&& (inr as usize) < (iend as usize)
|
||||
{
|
||||
let with_one = previous_match.price
|
||||
+ unsafe { raw_literals_cost(inr, 1, opt_state, opt_level) } as c_int
|
||||
+ (unsafe { lit_length_price(1, opt_state, opt_level) } as c_int
|
||||
- unsafe { lit_length_price(0, opt_state, opt_level) } as c_int);
|
||||
let with_more = price
|
||||
+ unsafe { raw_literals_cost(inr, 1, opt_state, opt_level) } as c_int
|
||||
+ (unsafe { lit_length_price(literal_length + 1, opt_state, opt_level) }
|
||||
as c_int
|
||||
- unsafe { lit_length_price(literal_length, opt_state, opt_level) }
|
||||
as c_int);
|
||||
if with_one < with_more
|
||||
&& with_one < unsafe { (*opt.add((cur + 1) as usize)).price }
|
||||
{
|
||||
let previous_index = cur - previous_match.mlen;
|
||||
let reps = new_rep(
|
||||
unsafe { (*opt.add(previous_index as usize)).rep },
|
||||
previous_match.off,
|
||||
unsafe { (*opt.add(previous_index as usize)).litlen == 0 },
|
||||
);
|
||||
unsafe {
|
||||
(*opt.add((cur + 1) as usize)).rep = reps;
|
||||
(*opt.add((cur + 1) as usize)).mlen = previous_match.mlen;
|
||||
(*opt.add((cur + 1) as usize)).off = previous_match.off;
|
||||
(*opt.add((cur + 1) as usize)).litlen = 1;
|
||||
(*opt.add((cur + 1) as usize)).price = with_one;
|
||||
}
|
||||
if last_pos < cur + 1 {
|
||||
last_pos = cur + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let selected = unsafe { *opt.add(cur as usize) };
|
||||
if selected.litlen == 0 {
|
||||
let previous_index = cur - selected.mlen;
|
||||
let reps = new_rep(
|
||||
unsafe { (*opt.add(previous_index as usize)).rep },
|
||||
selected.off,
|
||||
unsafe { (*opt.add(previous_index as usize)).litlen == 0 },
|
||||
);
|
||||
unsafe { (*opt.add(cur as usize)).rep = reps };
|
||||
}
|
||||
if (inr as usize) > (ilimit as usize) {
|
||||
cur += 1;
|
||||
continue;
|
||||
}
|
||||
if cur == last_pos {
|
||||
break;
|
||||
}
|
||||
if opt_level == 0
|
||||
&& unsafe { (*opt.add((cur + 1) as usize)).price }
|
||||
<= unsafe { (*opt.add(cur as usize)).price } + (BITCOST_MULTIPLIER / 2) as c_int
|
||||
{
|
||||
cur += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let previous_price = unsafe { (*opt.add(cur as usize)).price };
|
||||
let base_price =
|
||||
previous_price + unsafe { lit_length_price(0, opt_state, opt_level) } as c_int;
|
||||
let current_rep = unsafe { (*opt.add(cur as usize)).rep };
|
||||
let mut candidates = unsafe {
|
||||
get_all_matches(
|
||||
state,
|
||||
&mut next_to_update3,
|
||||
inr,
|
||||
iend,
|
||||
¤t_rep,
|
||||
u32::from(selected.litlen == 0),
|
||||
min_match,
|
||||
dict_mode,
|
||||
min_match,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
ldm_process_match(
|
||||
&mut opt_ldm,
|
||||
matches,
|
||||
&mut candidates,
|
||||
ptr_diff(inr, istart) as u32,
|
||||
ptr_diff(iend, inr) as u32,
|
||||
min_match,
|
||||
)
|
||||
};
|
||||
if candidates == 0 {
|
||||
cur += 1;
|
||||
continue;
|
||||
}
|
||||
let longest = unsafe { (*matches.add(candidates as usize - 1)).len };
|
||||
if longest > sufficient_len
|
||||
|| cur + longest >= ZSTD_OPT_NUM
|
||||
|| ptr_diff(iend, inr) <= longest as usize
|
||||
{
|
||||
last_stretch.mlen = longest;
|
||||
last_stretch.off = unsafe { (*matches.add(candidates as usize - 1)).off };
|
||||
last_stretch.litlen = 0;
|
||||
last_pos = cur + longest;
|
||||
let mut path_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
|
||||
unsafe {
|
||||
resolve_shortest_path(
|
||||
state,
|
||||
seq_store,
|
||||
&mut path_rep,
|
||||
istart,
|
||||
iend,
|
||||
&mut anchor,
|
||||
&mut ip,
|
||||
cur,
|
||||
last_pos,
|
||||
last_stretch,
|
||||
opt_level,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
*reps = path_rep[0];
|
||||
*reps.add(1) = path_rep[1];
|
||||
*reps.add(2) = path_rep[2];
|
||||
}
|
||||
resolved_early = true;
|
||||
cur = last_pos;
|
||||
break;
|
||||
}
|
||||
|
||||
for match_index in 0..candidates as usize {
|
||||
let candidate = unsafe { *matches.add(match_index) };
|
||||
let start_ml = if match_index > 0 {
|
||||
unsafe { (*matches.add(match_index - 1)).len + 1 }
|
||||
} else {
|
||||
min_match
|
||||
};
|
||||
let mut match_length = candidate.len;
|
||||
loop {
|
||||
let position = cur + match_length;
|
||||
let match_price = unsafe {
|
||||
get_match_price(candidate.off, match_length, opt_state, opt_level)
|
||||
} as c_int;
|
||||
let candidate_price = base_price + match_price;
|
||||
if position > last_pos
|
||||
|| candidate_price < unsafe { (*opt.add(position as usize)).price }
|
||||
{
|
||||
while last_pos < position {
|
||||
last_pos += 1;
|
||||
unsafe {
|
||||
(*opt.add(last_pos as usize)).price = ZSTD_MAX_PRICE;
|
||||
(*opt.add(last_pos as usize)).litlen = 1;
|
||||
}
|
||||
}
|
||||
unsafe {
|
||||
(*opt.add(position as usize)).mlen = match_length;
|
||||
(*opt.add(position as usize)).off = candidate.off;
|
||||
(*opt.add(position as usize)).litlen = 0;
|
||||
(*opt.add(position as usize)).price = candidate_price;
|
||||
}
|
||||
} else if opt_level == 0 {
|
||||
break;
|
||||
}
|
||||
if match_length == start_ml {
|
||||
break;
|
||||
}
|
||||
match_length -= 1;
|
||||
}
|
||||
}
|
||||
unsafe { (*opt.add((last_pos + 1) as usize)).price = ZSTD_MAX_PRICE };
|
||||
cur += 1;
|
||||
}
|
||||
|
||||
if cur == last_pos && !resolved_early {
|
||||
/* The previous loop reached its end without an early path. */
|
||||
last_stretch = unsafe { *opt.add(last_pos as usize) };
|
||||
let path_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
|
||||
let mut next_rep = path_rep;
|
||||
let path_cur = last_pos.wrapping_sub(last_stretch.mlen);
|
||||
unsafe {
|
||||
resolve_shortest_path(
|
||||
state,
|
||||
seq_store,
|
||||
&mut next_rep,
|
||||
istart,
|
||||
iend,
|
||||
&mut anchor,
|
||||
&mut ip,
|
||||
path_cur,
|
||||
last_pos,
|
||||
last_stretch,
|
||||
opt_level,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
*reps = next_rep[0];
|
||||
*reps.add(1) = next_rep[1];
|
||||
*reps.add(2) = next_rep[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
ptr_diff(iend, anchor)
|
||||
}
|
||||
|
||||
unsafe fn init_stats_ultra(
|
||||
state: &mut ZSTD_RustOptState,
|
||||
seq_store: *mut SeqStore_t,
|
||||
reps: *const u32,
|
||||
src: *const u8,
|
||||
src_size: usize,
|
||||
) {
|
||||
let mut temporary_rep = [unsafe { *reps }, unsafe { *reps.add(1) }, unsafe {
|
||||
*reps.add(2)
|
||||
}];
|
||||
unsafe {
|
||||
compress_block_opt_generic(
|
||||
state,
|
||||
seq_store,
|
||||
temporary_rep.as_mut_ptr(),
|
||||
src,
|
||||
src_size,
|
||||
2,
|
||||
DICT_NO_DICT,
|
||||
);
|
||||
}
|
||||
let store = unsafe { &mut *seq_store };
|
||||
store.sequences = store.sequencesStart;
|
||||
store.lit = store.litStart;
|
||||
store.longLengthType = 0;
|
||||
unsafe {
|
||||
*state.window_base = (*state.window_base).wrapping_sub(src_size);
|
||||
*state.window_dict_limit = (*state.window_dict_limit).wrapping_add(src_size as u32);
|
||||
*state.window_low_limit = *state.window_dict_limit;
|
||||
*state.next_to_update = *state.window_dict_limit;
|
||||
}
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_opt_updateTree(
|
||||
state: *mut ZSTD_RustOptState,
|
||||
ip: *const c_void,
|
||||
iend: *const c_void,
|
||||
mls: u32,
|
||||
dict_mode: c_int,
|
||||
) {
|
||||
let state = unsafe { &mut *state };
|
||||
unsafe {
|
||||
ZSTD_rust_opt_updateTreeInternal(
|
||||
(state as *mut ZSTD_RustOptState).cast::<OptTreePrefix>(),
|
||||
ip,
|
||||
iend,
|
||||
mls,
|
||||
c_int::from(dict_mode == DICT_EXT),
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_compressBlock_opt(
|
||||
state: *mut ZSTD_RustOptState,
|
||||
seq_store: *mut c_void,
|
||||
reps: *mut u32,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
opt_level: c_int,
|
||||
dict_mode: c_int,
|
||||
) -> usize {
|
||||
let state = unsafe { &mut *state };
|
||||
unsafe {
|
||||
compress_block_opt_generic(
|
||||
state,
|
||||
seq_store.cast::<SeqStore_t>(),
|
||||
reps,
|
||||
src.cast::<u8>(),
|
||||
src_size,
|
||||
opt_level,
|
||||
dict_mode,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_initStats_ultra(
|
||||
state: *mut ZSTD_RustOptState,
|
||||
seq_store: *mut c_void,
|
||||
reps: *const u32,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
) {
|
||||
let state = unsafe { &mut *state };
|
||||
unsafe {
|
||||
init_stats_ultra(
|
||||
state,
|
||||
seq_store.cast::<SeqStore_t>(),
|
||||
reps,
|
||||
src.cast::<u8>(),
|
||||
src_size,
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_compressBlock_btultra2(
|
||||
state: *mut ZSTD_RustOptState,
|
||||
seq_store: *mut c_void,
|
||||
reps: *mut u32,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
) -> usize {
|
||||
let state = unsafe { &mut *state };
|
||||
let store = unsafe { &*seq_store.cast::<SeqStore_t>() };
|
||||
let current = index_from(state.base, src.cast::<u8>());
|
||||
let should_seed = unsafe { (*state.opt).litLengthSum == 0 }
|
||||
&& store.sequences == store.sequencesStart
|
||||
&& unsafe { *state.window_dict_limit == *state.window_low_limit }
|
||||
&& current == unsafe { *state.window_dict_limit }
|
||||
&& src_size > ZSTD_PREDEF_THRESHOLD;
|
||||
if should_seed {
|
||||
unsafe { init_stats_ultra(state, seq_store.cast(), reps, src.cast(), src_size) };
|
||||
}
|
||||
unsafe {
|
||||
compress_block_opt_generic(
|
||||
state,
|
||||
seq_store.cast::<SeqStore_t>(),
|
||||
reps,
|
||||
src.cast::<u8>(),
|
||||
src_size,
|
||||
2,
|
||||
DICT_NO_DICT,
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user