ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
b0488f285ad9883f489ebc915648488b7027dc66
zstd-rs/rust/src/zstd_opt.rs
T
ddidderr 98cca9aa0f fix(compress): guard dictionary repcode coordinates 
Attached-dictionary matchers derive dictionary and prefix pointers from
32-bit repcode coordinates.  A stale offset in the Rust double-fast
post-match path could wrap the subtraction to a large index, while the
C-compatible overlap predicate's intentional unsigned arithmetic then
classified that index as safe.  Dictionary training consequently reached an
invalid read in the multithreaded FastCover workers.

Centralize the attached-dictionary coordinate check before pointer formation:
reject disabled or wrapped offsets, keep repcodes no later than the current
position, require four-byte dictionary matches to remain in the translated
dictionary interval, and preserve the active-prefix coordinate region.  Apply
the same checked overlap semantics in the optimal parser, where the current
position is available to reject wrapped repcodes.  Focused tests cover the
crashing coordinate, dictionary boundaries, prefix boundaries, and valid
active-prefix repcodes.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --lib zstd_double_fast::tests` -- passed (4 tests)
- `cargo test --manifest-path rust/Cargo.toml --lib zstd_opt::tests::index_overlap_check` -- passed (2 tests)
- `cargo test --manifest-path rust/Cargo.toml --all-targets` -- passed (400 tests)
- `cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings` -- passed
- `rustfmt +nightly --edition 2021 rust/src/zstd_double_fast.rs rust/src/zstd_opt.rs --check` -- passed
- `make -C programs -j2 zstd` -- passed
- `./programs/zstd --train -B2K tests/tmpCorpusHighCompress -o /tmp/zstd-rust-dict-high-fixed` -- passed
- `make -C tests -j2 test-zstd` -- passed
- Full `cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings` remains blocked by the pre-existing `manual_repeat_n` lint at `rust/src/zstd_compress.rs:2351`.
2026-07-18 17:13:27 +02:00

1932 lines
64 KiB
Rust
Raw Blame History

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
#![allow(clippy::not_unsafe_ptr_arg_deref)]
//! Optimal block parsing for the binary-tree compression strategies.
//!
//! The compression context remains private to C. `zstd_opt.c` projects the
//! fields used by this module into `ZSTD_RustOptState`; all buffers and tables
//! remain owned by the caller. This keeps the ABI independent of the private
//! `ZSTD_MatchState_t` layout while moving the price model, match enumeration,
//! optimal parse, and sequence emission into Rust.
use crate::bits::ZSTD_highbit32;
use crate::common::{LL_BITS, MAX_LIT, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS};
use crate::mem::{
MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST,
};
use std::cmp::min;
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::{c_int, c_uint};
use std::ptr;
const ZSTD_REP_NUM: usize = 3;
const HASH_READ_SIZE: usize = 8;
const ZSTD_OPT_NUM: u32 = 1 << 12;
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
const ZSTD_LITFREQ_ADD: u32 = 2;
const ZSTD_MAX_PRICE: i32 = 1 << 30;
const ZSTD_PREDEF_THRESHOLD: usize = 8;
const BITCOST_ACCURACY: u32 = 8;
const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY;
const OFFSET_OFFBASE: u32 = ZSTD_REP_NUM as u32;
const PRIME3BYTES: u32 = 506_832_829;
const ZOP_DYNAMIC: c_int = 0;
const ZOP_PREDEF: c_int = 1;
const ZSTD_PS_DISABLE: c_int = 2;
const DICT_NO_DICT: c_int = 0;
const DICT_EXT: c_int = 1;
const DICT_MATCH_STATE: c_int = 2;
const LL_CODE: [u8; 64] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20,
20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
];
#[allow(dead_code)]
const ML_CODE: [u8; 149] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38,
38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42,
];
#[repr(C)]
#[derive(Clone, Copy)]
struct SeqDef {
offBase: u32,
litLength: u16,
mlBase: u16,
}
#[repr(C)]
struct SeqStore_t {
sequencesStart: *mut SeqDef,
sequences: *mut SeqDef,
litStart: *mut u8,
lit: *mut u8,
llCode: *mut u8,
mlCode: *mut u8,
ofCode: *mut u8,
maxNbSeq: usize,
maxNbLit: usize,
longLengthType: c_int,
longLengthPos: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Default)]
struct ZSTD_match_t {
off: u32,
len: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Default)]
struct ZSTD_optimal_t {
price: c_int,
off: u32,
mlen: u32,
litlen: u32,
rep: [u32; ZSTD_REP_NUM],
}
#[repr(C)]
#[derive(Clone, Copy, Default)]
struct RawSeq {
offset: u32,
litLength: u32,
matchLength: u32,
}
#[repr(C)]
#[derive(Clone, Copy)]
struct RawSeqStore {
seq: *const RawSeq,
pos: usize,
posInSequence: usize,
size: usize,
capacity: usize,
}
impl Default for RawSeqStore {
fn default() -> Self {
Self {
seq: ptr::null(),
pos: 0,
posInSequence: 0,
size: 0,
capacity: 0,
}
}
}
#[repr(C)]
struct OptState {
litFreq: *mut c_uint,
litLengthFreq: *mut c_uint,
matchLengthFreq: *mut c_uint,
offCodeFreq: *mut c_uint,
matchTable: *mut ZSTD_match_t,
priceTable: *mut ZSTD_optimal_t,
litSum: u32,
litLengthSum: u32,
matchLengthSum: u32,
offCodeSum: u32,
litSumBasePrice: u32,
litLengthSumBasePrice: u32,
matchLengthSumBasePrice: u32,
offCodeSumBasePrice: u32,
priceType: c_int,
symbolCosts: *const c_void,
literalCompressionMode: c_int,
}
/* The first fields intentionally match zstd_opt_tree's public leaf view. */
#[repr(C)]
struct OptTreePrefix {
hash_table: *mut u32,
chain_table: *mut u32,
base: *const u8,
dict_base: *const u8,
dict_limit: u32,
low_limit: u32,
loaded_dict_end: u32,
next_to_update: *mut u32,
hash_log: u32,
chain_log: u32,
search_log: u32,
window_log: u32,
use_c_predict: c_int,
}
/// Private-state projection populated by `lib/compress/zstd_opt.c`.
#[repr(C)]
pub struct ZSTD_RustOptState {
hash_table: *mut u32,
chain_table: *mut u32,
base: *const u8,
dict_base: *const u8,
dict_limit: u32,
low_limit: u32,
loaded_dict_end: u32,
next_to_update: *mut u32,
hash_log: u32,
chain_log: u32,
search_log: u32,
window_log: u32,
use_c_predict: c_int,
hash_table3: *mut u32,
hash_log3: u32,
next_src: *const u8,
min_match: u32,
target_length: u32,
opt: *mut OptState,
dict_match_state: *const ZSTD_RustOptState,
ldm_seq_store: *const RawSeqStore,
window_base: *mut *const u8,
window_dict_limit: *mut u32,
window_low_limit: *mut u32,
huf_ctable: *const usize,
huf_repeat_valid: c_int,
fse_litlength_ctable: *const u32,
fse_matchlength_ctable: *const u32,
fse_offcode_ctable: *const u32,
}
#[repr(C)]
#[derive(Clone, Copy)]
struct FseSymbolCompressionTransform {
deltaFindState: c_int,
deltaNbBits: u32,
}
extern "C" {
fn ZSTD_rust_opt_updateTreeInternal(
state: *mut OptTreePrefix,
ip: *const c_void,
iend: *const c_void,
mls: u32,
extDict: c_int,
);
fn HUF_getNbBitsFromCTable(ctable: *const usize, symbolValue: u32) -> u32;
}
#[inline]
fn ptr_diff(left: *const u8, right: *const u8) -> usize {
(left as usize).wrapping_sub(right as usize)
}
#[inline]
fn index_from(base: *const u8, value: *const u8) -> u32 {
ptr_diff(value, base) as u32
}
#[inline]
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
let input_start = input;
let word_size = size_of::<usize>();
while ptr_diff(input_limit, input) >= word_size {
let difference =
unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
if difference != 0 {
let common = if MEM_isLittleEndian() {
difference.trailing_zeros()
} else {
difference.leading_zeros()
} as usize
/ 8;
return ptr_diff(input, input_start) + common;
}
input = input.wrapping_add(word_size);
matched = matched.wrapping_add(word_size);
}
if MEM_64bits()
&& ptr_diff(input_limit, input) >= 4
&& unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
{
input = input.wrapping_add(4);
matched = matched.wrapping_add(4);
}
if ptr_diff(input_limit, input) >= 2
&& unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
{
input = input.wrapping_add(2);
matched = matched.wrapping_add(2);
}
if ptr_diff(input, input_limit) != 0 && unsafe { *matched == *input } {
input = input.wrapping_add(1);
}
ptr_diff(input, input_start)
}
unsafe fn count_2segments(
input: *const u8,
matched: *const u8,
input_end: *const u8,
match_end: *const u8,
input_start: *const u8,
) -> usize {
let match_remaining = ptr_diff(match_end, matched);
let input_remaining = ptr_diff(input_end, input);
let first_end = input.wrapping_add(min(match_remaining, input_remaining));
let first_count = unsafe { count(input, matched, first_end) };
if matched.wrapping_add(first_count) != match_end {
return first_count;
}
first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
}
#[inline]
unsafe fn hash_ptr(input: *const u8, hbits: u32, mls: u32) -> usize {
let hash_shift32 = |value: u32| {
if hbits == 0 {
0
} else {
(value >> (32 - hbits)) as usize
}
};
let hash_shift64 = |value: u64| {
if hbits == 0 {
0
} else {
(value >> (64 - hbits)) as usize
}
};
match mls {
5 => hash_shift64(
unsafe { MEM_readLE64(input.cast::<c_void>()) }
.wrapping_shl(24)
.wrapping_mul(889_523_592_379),
),
6 => hash_shift64(
unsafe { MEM_readLE64(input.cast::<c_void>()) }
.wrapping_shl(16)
.wrapping_mul(227_718_039_650_203),
),
7 => hash_shift64(
unsafe { MEM_readLE64(input.cast::<c_void>()) }
.wrapping_shl(8)
.wrapping_mul(58_295_818_150_454_627),
),
8 => hash_shift64(
unsafe { MEM_readLE64(input.cast::<c_void>()) }.wrapping_mul(0xCF1B_BCDC_B7A5_6463),
),
_ => hash_shift32(
unsafe { MEM_readLE32(input.cast::<c_void>()) }.wrapping_mul(2_654_435_761),
),
}
}
#[inline]
unsafe fn hash3_ptr(input: *const u8, hbits: u32) -> usize {
if hbits == 0 {
0
} else {
let value = unsafe { MEM_readLE32(input.cast::<c_void>()) };
(value.wrapping_shl(8).wrapping_mul(PRIME3BYTES) >> (32 - hbits)) as usize
}
}
#[inline]
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
unsafe { *table.add(index) }
}
#[inline]
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
unsafe { *table.add(index) = value };
}
#[inline]
fn lowest_match_index(state: &ZSTD_RustOptState, current: u32) -> u32 {
let max_distance = 1u32.wrapping_shl(state.window_log);
let within_window = if current.wrapping_sub(state.low_limit) > max_distance {
current.wrapping_sub(max_distance)
} else {
state.low_limit
};
if state.loaded_dict_end != 0 {
state.low_limit
} else {
within_window
}
}
#[inline]
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32, current: u32) -> bool {
if rep_index > current {
return false;
}
rep_index >= prefix_lowest_index
|| rep_index
.checked_add(4)
.is_some_and(|rep_end| rep_end <= prefix_lowest_index)
}
#[inline]
unsafe fn read_min_match(input: *const u8, length: u32) -> u32 {
let value = unsafe { MEM_read32(input.cast::<c_void>()) };
if length == 3 && MEM_isLittleEndian() {
value << 8
} else if length == 3 {
value >> 8
} else {
value
}
}
#[inline]
fn ll_code(value: u32) -> usize {
if value > 63 {
(ZSTD_highbit32(value) + 19) as usize
} else {
LL_CODE[value as usize] as usize
}
}
#[inline]
fn ml_code(value: u32) -> usize {
match value {
0..=31 => value as usize,
32..=33 => 32,
34..=35 => 33,
36..=37 => 34,
38..=39 => 35,
40..=43 => 36,
44..=47 => 37,
48..=51 => 38,
52..=55 => 39,
56..=71 => 40,
72..=87 => 41,
88..=127 => 42,
_ => (ZSTD_highbit32(value) + 36) as usize,
}
}
#[inline]
fn bit_weight(stat: u32) -> u32 {
ZSTD_highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER
}
#[inline]
fn frac_weight(raw_stat: u32) -> u32 {
let stat = raw_stat.wrapping_add(1);
let high_bit = ZSTD_highbit32(stat);
let base = high_bit * BITCOST_MULTIPLIER;
base + (stat << BITCOST_ACCURACY >> high_bit)
}
#[inline]
fn weight(stat: u32, opt_level: c_int) -> u32 {
if opt_level >= 2 {
frac_weight(stat)
} else {
bit_weight(stat)
}
}
unsafe fn sum_u32(table: *const c_uint, count: usize) -> u32 {
let mut total: u32 = 0;
for index in 0..count {
total = total.wrapping_add(unsafe { *table.add(index) });
}
total
}
unsafe fn downscale_stats(
table: *mut c_uint,
last_index: usize,
shift: u32,
base_one: bool,
) -> u32 {
let mut sum: u32 = 0;
for index in 0..=last_index {
let old = unsafe { *table.add(index) };
/* `base_0possible` still gives every observed symbol a count of one.
* Only symbols which were absent from the source may remain at zero. */
let base = if base_one { 1 } else { u32::from(old != 0) };
let value = base + (old >> shift);
unsafe { *table.add(index) = value };
sum = sum.wrapping_add(value);
}
sum
}
unsafe fn scale_stats(table: *mut c_uint, last_index: usize, log_target: u32) -> u32 {
let previous = unsafe { sum_u32(table, last_index + 1) };
let factor = previous >> log_target;
if factor <= 1 {
previous
} else {
unsafe { downscale_stats(table, last_index, ZSTD_highbit32(factor), true) }
}
}
unsafe fn fse_max_nb_bits(table: *const u32, symbol: usize) -> u32 {
let table_log = unsafe { MEM_read16(table.cast::<c_void>()) } as usize;
let transform_offset = 1 + if table_log == 0 {
1
} else {
1 << (table_log - 1)
};
let transform = unsafe {
table
.add(transform_offset)
.cast::<FseSymbolCompressionTransform>()
.add(symbol)
.read()
};
(transform.deltaNbBits.wrapping_add(0xFFFF)) >> 16
}
#[inline]
fn compressed_literals(opt: &OptState) -> bool {
opt.literalCompressionMode != ZSTD_PS_DISABLE
}
unsafe fn set_base_prices(opt: &mut OptState, opt_level: c_int) {
if compressed_literals(opt) {
opt.litSumBasePrice = weight(opt.litSum, opt_level);
}
opt.litLengthSumBasePrice = weight(opt.litLengthSum, opt_level);
opt.matchLengthSumBasePrice = weight(opt.matchLengthSum, opt_level);
opt.offCodeSumBasePrice = weight(opt.offCodeSum, opt_level);
}
unsafe fn rescale_freqs(
state: &ZSTD_RustOptState,
src: *const u8,
src_size: usize,
opt_level: c_int,
) {
let opt = unsafe { &mut *state.opt };
let compressed = compressed_literals(opt);
opt.priceType = ZOP_DYNAMIC;
if opt.litLengthSum == 0 {
if src_size <= ZSTD_PREDEF_THRESHOLD {
opt.priceType = ZOP_PREDEF;
}
if state.huf_repeat_valid == 0 {
/* No valid dictionary table: seed from the first source block. */
if compressed {
for index in 0..=MAX_LIT {
unsafe { *opt.litFreq.add(index) = 0 };
}
for index in 0..src_size {
let byte = unsafe { *src.add(index) } as usize;
unsafe { *opt.litFreq.add(byte) = (*opt.litFreq.add(byte)).wrapping_add(1) };
}
opt.litSum = unsafe { downscale_stats(opt.litFreq, MAX_LIT, 8, false) };
}
let base_ll: [u32; MAX_LL + 1] = [
4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
];
unsafe {
ptr::copy_nonoverlapping(base_ll.as_ptr(), opt.litLengthFreq, base_ll.len());
}
opt.litLengthSum = unsafe { sum_u32(base_ll.as_ptr(), base_ll.len()) };
for index in 0..=MAX_ML {
unsafe { *opt.matchLengthFreq.add(index) = 1 };
}
opt.matchLengthSum = (MAX_ML + 1) as u32;
let base_of: [u32; MAX_OFF + 1] = [
6, 2, 1, 1, 2, 3, 4, 4, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1,
];
unsafe {
ptr::copy_nonoverlapping(base_of.as_ptr(), opt.offCodeFreq, base_of.len());
}
opt.offCodeSum = unsafe { sum_u32(base_of.as_ptr(), base_of.len()) };
} else {
/* Dictionary-seeded tables are projected as leaf pointers by C. */
if compressed && !state.huf_ctable.is_null() {
opt.litSum = 0;
for lit in 0..=MAX_LIT {
let bit_cost = unsafe { HUF_getNbBitsFromCTable(state.huf_ctable, lit as u32) };
let value = if bit_cost != 0 {
1u32 << (11 - bit_cost)
} else {
1
};
unsafe { *opt.litFreq.add(lit) = value };
opt.litSum = opt.litSum.wrapping_add(value);
}
}
if !state.fse_litlength_ctable.is_null() {
opt.litLengthSum = 0;
for symbol in 0..=MAX_LL {
let bits = unsafe { fse_max_nb_bits(state.fse_litlength_ctable, symbol) };
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
unsafe { *opt.litLengthFreq.add(symbol) = value };
opt.litLengthSum = opt.litLengthSum.wrapping_add(value);
}
}
if !state.fse_matchlength_ctable.is_null() {
opt.matchLengthSum = 0;
for symbol in 0..=MAX_ML {
let bits = unsafe { fse_max_nb_bits(state.fse_matchlength_ctable, symbol) };
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
unsafe { *opt.matchLengthFreq.add(symbol) = value };
opt.matchLengthSum = opt.matchLengthSum.wrapping_add(value);
}
}
if !state.fse_offcode_ctable.is_null() {
opt.offCodeSum = 0;
for symbol in 0..=MAX_OFF {
let bits = unsafe { fse_max_nb_bits(state.fse_offcode_ctable, symbol) };
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
unsafe { *opt.offCodeFreq.add(symbol) = value };
opt.offCodeSum = opt.offCodeSum.wrapping_add(value);
}
}
}
} else {
if compressed {
opt.litSum = unsafe { scale_stats(opt.litFreq, MAX_LIT, 12) };
}
opt.litLengthSum = unsafe { scale_stats(opt.litLengthFreq, MAX_LL, 11) };
opt.matchLengthSum = unsafe { scale_stats(opt.matchLengthFreq, MAX_ML, 11) };
opt.offCodeSum = unsafe { scale_stats(opt.offCodeFreq, MAX_OFF, 11) };
}
unsafe { set_base_prices(opt, opt_level) };
}
unsafe fn raw_literals_cost(
literals: *const u8,
lit_length: u32,
opt: &OptState,
opt_level: c_int,
) -> u32 {
if lit_length == 0 {
return 0;
}
if !compressed_literals(opt) {
return lit_length.wrapping_mul(8).wrapping_mul(BITCOST_MULTIPLIER);
}
if opt.priceType == ZOP_PREDEF {
return lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER);
}
let mut price = opt.litSumBasePrice.wrapping_mul(lit_length);
let max_price = opt.litSumBasePrice.wrapping_sub(BITCOST_MULTIPLIER);
for index in 0..lit_length as usize {
let literal = unsafe { *literals.add(index) } as usize;
let frequency = unsafe { *opt.litFreq.add(literal) };
price = price.wrapping_sub(min(weight(frequency, opt_level), max_price));
}
price
}
unsafe fn lit_length_price(lit_length: u32, opt: &OptState, opt_level: c_int) -> u32 {
if opt.priceType == ZOP_PREDEF {
return weight(lit_length, opt_level);
}
if lit_length as usize == ZSTD_BLOCKSIZE_MAX {
return BITCOST_MULTIPLIER
+ unsafe { lit_length_price((ZSTD_BLOCKSIZE_MAX - 1) as u32, opt, opt_level) };
}
let code = ll_code(lit_length);
(LL_BITS[code] as u32 * BITCOST_MULTIPLIER) + opt.litLengthSumBasePrice
- weight(unsafe { *opt.litLengthFreq.add(code) }, opt_level)
}
unsafe fn get_match_price(
off_base: u32,
match_length: u32,
opt: &OptState,
opt_level: c_int,
) -> u32 {
let off_code = ZSTD_highbit32(off_base);
let ml_base = match_length - MINMATCH as u32;
if opt.priceType == ZOP_PREDEF {
return weight(ml_base, opt_level) + (16u32 + off_code).wrapping_mul(BITCOST_MULTIPLIER);
}
let mut price = off_code * BITCOST_MULTIPLIER + opt.offCodeSumBasePrice
- weight(
unsafe { *opt.offCodeFreq.add(off_code as usize) },
opt_level,
);
if opt_level < 2 && off_code >= 20 {
price += (off_code - 19) * 2 * BITCOST_MULTIPLIER;
}
let code = ml_code(ml_base);
price += ML_BITS[code] as u32 * BITCOST_MULTIPLIER + opt.matchLengthSumBasePrice
- weight(unsafe { *opt.matchLengthFreq.add(code) }, opt_level);
price + BITCOST_MULTIPLIER / 5
}
unsafe fn update_stats(
opt: &mut OptState,
lit_length: u32,
literals: *const u8,
off_base: u32,
match_length: u32,
) {
if compressed_literals(opt) {
for index in 0..lit_length as usize {
let literal = unsafe { *literals.add(index) } as usize;
let frequency = unsafe { opt.litFreq.add(literal) };
unsafe { *frequency = (*frequency).wrapping_add(ZSTD_LITFREQ_ADD) };
}
opt.litSum = opt
.litSum
.wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD));
}
let ll = ll_code(lit_length);
unsafe { *opt.litLengthFreq.add(ll) = (*opt.litLengthFreq.add(ll)).wrapping_add(1) };
opt.litLengthSum = opt.litLengthSum.wrapping_add(1);
let off = ZSTD_highbit32(off_base) as usize;
unsafe { *opt.offCodeFreq.add(off) = (*opt.offCodeFreq.add(off)).wrapping_add(1) };
opt.offCodeSum = opt.offCodeSum.wrapping_add(1);
let ml = ml_code(match_length - MINMATCH as u32);
unsafe { *opt.matchLengthFreq.add(ml) = (*opt.matchLengthFreq.add(ml)).wrapping_add(1) };
opt.matchLengthSum = opt.matchLengthSum.wrapping_add(1);
}
#[inline]
fn offset_to_offbase(offset: u32) -> u32 {
offset.wrapping_add(OFFSET_OFFBASE)
}
#[inline]
fn repcode_to_offbase(rep_code: u32) -> u32 {
rep_code
}
#[inline]
fn new_rep(mut rep: [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] {
if off_base > OFFSET_OFFBASE {
rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = off_base - OFFSET_OFFBASE;
} else {
let rep_code = off_base - 1 + u32::from(ll0);
if rep_code != 0 {
let current = if rep_code == ZSTD_REP_NUM as u32 {
rep[0].wrapping_sub(1)
} else {
rep[rep_code as usize]
};
if rep_code >= 2 {
rep[2] = rep[1];
}
rep[1] = rep[0];
rep[0] = current;
}
}
rep
}
unsafe fn store_seq(
seq_store: *mut SeqStore_t,
lit_length: usize,
literals: *const u8,
off_base: u32,
match_length: usize,
) {
let store = unsafe { &mut *seq_store };
let sequence = store.sequences;
let index =
ptr_diff(sequence.cast::<u8>(), store.sequencesStart.cast::<u8>()) / size_of::<SeqDef>();
if lit_length != 0 {
unsafe { ptr::copy_nonoverlapping(literals, store.lit, lit_length) };
}
store.lit = store.lit.wrapping_add(lit_length);
if lit_length > u16::MAX as usize {
store.longLengthType = 1;
store.longLengthPos = index as u32;
}
unsafe {
(*sequence).litLength = lit_length as u16;
(*sequence).offBase = off_base;
}
let ml_base = match_length - MINMATCH;
if ml_base > u16::MAX as usize {
store.longLengthType = 2;
store.longLengthPos = index as u32;
}
unsafe { (*sequence).mlBase = ml_base as u16 };
store.sequences = store.sequences.wrapping_add(1);
}
unsafe fn skip_raw_seq_store_bytes(store: &mut RawSeqStore, nb_bytes: usize) {
let mut current = (store.posInSequence + nb_bytes) as u32;
while current != 0 && store.pos < store.size {
let sequence = unsafe { store.seq.add(store.pos).read() };
let sequence_size = sequence.litLength.wrapping_add(sequence.matchLength);
if current >= sequence_size {
current -= sequence_size;
store.pos += 1;
} else {
store.posInSequence = current as usize;
break;
}
}
if current == 0 || store.pos == store.size {
store.posInSequence = 0;
}
}
#[derive(Clone, Copy)]
struct OptLdm {
seq_store: RawSeqStore,
start_pos: u32,
end_pos: u32,
offset: u32,
}
unsafe fn ldm_next_match(opt_ldm: &mut OptLdm, current_pos: u32, block_remaining: u32) {
if opt_ldm.seq_store.size == 0 || opt_ldm.seq_store.pos >= opt_ldm.seq_store.size {
opt_ldm.start_pos = u32::MAX;
opt_ldm.end_pos = u32::MAX;
return;
}
let sequence = unsafe { opt_ldm.seq_store.seq.add(opt_ldm.seq_store.pos).read() };
let block_end = current_pos.wrapping_add(block_remaining);
let literals_remaining =
(sequence.litLength as usize).saturating_sub(opt_ldm.seq_store.posInSequence) as u32;
let match_remaining = if literals_remaining == 0 {
sequence
.matchLength
.wrapping_sub(opt_ldm.seq_store.posInSequence as u32 - sequence.litLength)
} else {
sequence.matchLength
};
if literals_remaining >= block_remaining {
opt_ldm.start_pos = u32::MAX;
opt_ldm.end_pos = u32::MAX;
unsafe { skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, block_remaining as usize) };
return;
}
opt_ldm.start_pos = current_pos.wrapping_add(literals_remaining);
opt_ldm.end_pos = opt_ldm.start_pos.wrapping_add(match_remaining);
opt_ldm.offset = sequence.offset;
if opt_ldm.end_pos > block_end {
opt_ldm.end_pos = block_end;
unsafe {
skip_raw_seq_store_bytes(
&mut opt_ldm.seq_store,
block_end.wrapping_sub(current_pos) as usize,
)
};
} else {
unsafe {
skip_raw_seq_store_bytes(
&mut opt_ldm.seq_store,
(literals_remaining + match_remaining) as usize,
)
};
}
}
unsafe fn ldm_maybe_add_match(
matches: *mut ZSTD_match_t,
nb_matches: &mut u32,
opt_ldm: &OptLdm,
current_pos: u32,
min_match: u32,
) {
let position_diff = current_pos.wrapping_sub(opt_ldm.start_pos);
let candidate_length = opt_ldm
.end_pos
.wrapping_sub(opt_ldm.start_pos)
.wrapping_sub(position_diff);
if current_pos < opt_ldm.start_pos
|| current_pos >= opt_ldm.end_pos
|| candidate_length < min_match
{
return;
}
if *nb_matches == 0
|| (candidate_length > unsafe { (*matches.add(*nb_matches as usize - 1)).len }
&& *nb_matches < ZSTD_OPT_NUM)
{
let slot = unsafe { &mut *matches.add(*nb_matches as usize) };
slot.len = candidate_length;
slot.off = offset_to_offbase(opt_ldm.offset);
*nb_matches += 1;
}
}
unsafe fn ldm_process_match(
opt_ldm: &mut OptLdm,
matches: *mut ZSTD_match_t,
nb_matches: &mut u32,
current_pos: u32,
remaining: u32,
min_match: u32,
) {
if opt_ldm.seq_store.size == 0 || opt_ldm.seq_store.pos >= opt_ldm.seq_store.size {
return;
}
if current_pos >= opt_ldm.end_pos {
if current_pos > opt_ldm.end_pos {
unsafe {
skip_raw_seq_store_bytes(
&mut opt_ldm.seq_store,
current_pos.wrapping_sub(opt_ldm.end_pos) as usize,
)
};
}
unsafe { ldm_next_match(opt_ldm, current_pos, remaining) };
}
unsafe { ldm_maybe_add_match(matches, nb_matches, opt_ldm, current_pos, min_match) };
}
unsafe fn insert_and_find_first_index_hash3(
state: &ZSTD_RustOptState,
next_to_update3: &mut u32,
ip: *const u8,
) -> u32 {
let target = index_from(state.base, ip);
let hash = unsafe { hash3_ptr(ip, state.hash_log3) };
let mut index = *next_to_update3;
while index < target {
let position = state.base.wrapping_add(index as usize);
let position_hash = unsafe { hash3_ptr(position, state.hash_log3) };
unsafe { table_set(state.hash_table3, position_hash, index) };
index = index.wrapping_add(1);
}
*next_to_update3 = target;
unsafe { table_get(state.hash_table3, hash) }
}
unsafe fn insert_bt_and_get_all_matches(
state: &ZSTD_RustOptState,
next_to_update3: &mut u32,
ip: *const u8,
input_limit: *const u8,
dict_mode: c_int,
rep: &[u32; ZSTD_REP_NUM],
ll0: u32,
length_to_beat: u32,
mls: u32,
) -> u32 {
let sufficient_len = min(state.target_length, ZSTD_OPT_NUM - 1);
let current = index_from(state.base, ip);
let min_match = if mls == 3 { 3 } else { 4 };
let hash = unsafe { hash_ptr(ip, state.hash_log, mls) };
let mut match_index = unsafe { table_get(state.hash_table, hash) };
let bt_log = state.chain_log.wrapping_sub(1);
let bt_mask = (1u32 << bt_log).wrapping_sub(1);
let dict_limit = state.dict_limit;
let dict_end = state.dict_base.wrapping_add(dict_limit as usize);
let prefix_start = state.base.wrapping_add(dict_limit as usize);
let bt_low = current.saturating_sub(bt_mask);
let window_low = lowest_match_index(state, current);
let match_low = if window_low == 0 { 1 } else { window_low };
let mut smaller = unsafe { state.chain_table.add(2 * (current & bt_mask) as usize) };
let mut larger = unsafe { smaller.add(1) };
let mut dummy = 0u32;
let mut match_end_index = current.wrapping_add(HASH_READ_SIZE as u32 + 1);
let mut common_smaller = 0usize;
let mut common_larger = 0usize;
let mut best_length = length_to_beat.wrapping_sub(1) as usize;
let mut match_count = 0u32;
let mut nb_compares = 1u32 << state.search_log;
let dms = if dict_mode == DICT_MATCH_STATE {
debug_assert!(!state.dict_match_state.is_null());
unsafe { &*state.dict_match_state }
} else {
state
};
let dms_base = dms.base;
let dms_end = dms.next_src;
let dms_high_limit = if dict_mode == DICT_MATCH_STATE {
index_from(dms_base, dms_end)
} else {
0
};
let dms_low_limit = if dict_mode == DICT_MATCH_STATE {
dms.low_limit
} else {
0
};
let dms_index_delta = if dict_mode == DICT_MATCH_STATE {
window_low.wrapping_sub(dms_high_limit)
} else {
0
};
let dms_hash_log = if dict_mode == DICT_MATCH_STATE {
dms.hash_log
} else {
state.hash_log
};
let dms_bt_log = if dict_mode == DICT_MATCH_STATE {
dms.chain_log.wrapping_sub(1)
} else {
bt_log
};
let dms_bt_mask = if dict_mode == DICT_MATCH_STATE {
(1u32 << dms_bt_log).wrapping_sub(1)
} else {
0
};
let dms_bt_low = if dict_mode == DICT_MATCH_STATE
&& dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit)
{
dms_high_limit.wrapping_sub(dms_bt_mask)
} else {
dms_low_limit
};
for rep_code in ll0..(ZSTD_REP_NUM as u32 + ll0) {
let rep_offset = if rep_code == ZSTD_REP_NUM as u32 {
rep[0].wrapping_sub(1)
} else {
rep[rep_code as usize]
};
let rep_index = current.wrapping_sub(rep_offset);
let mut rep_len = 0usize;
if rep_offset.wrapping_sub(1) < current.wrapping_sub(dict_limit) {
if rep_index >= window_low
&& unsafe { read_min_match(ip, min_match) }
== unsafe { read_min_match(ip.wrapping_sub(rep_offset as usize), min_match) }
{
rep_len = unsafe {
count(
ip.wrapping_add(min_match as usize),
ip.wrapping_sub(rep_offset as usize)
.wrapping_add(min_match as usize),
input_limit,
) + min_match as usize
};
}
} else {
let rep_match = if dict_mode == DICT_MATCH_STATE {
dms_base.wrapping_add(rep_index.wrapping_sub(dms_index_delta) as usize)
} else {
state.dict_base.wrapping_add(rep_index as usize)
};
if dict_mode == DICT_EXT
&& rep_offset.wrapping_sub(1) < current.wrapping_sub(window_low)
&& index_overlap_check(dict_limit, rep_index, current)
&& unsafe { read_min_match(ip, min_match) }
== unsafe { read_min_match(rep_match, min_match) }
{
rep_len = unsafe {
count_2segments(
ip.wrapping_add(min_match as usize),
rep_match.wrapping_add(min_match as usize),
input_limit,
dict_end,
prefix_start,
) + min_match as usize
};
}
if dict_mode == DICT_MATCH_STATE
&& rep_offset.wrapping_sub(1)
< current.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta))
&& index_overlap_check(dict_limit, rep_index, current)
&& unsafe { read_min_match(ip, min_match) }
== unsafe { read_min_match(rep_match, min_match) }
{
rep_len = unsafe {
count_2segments(
ip.wrapping_add(min_match as usize),
rep_match.wrapping_add(min_match as usize),
input_limit,
dms_end,
prefix_start,
) + min_match as usize
};
}
}
if rep_len > best_length {
best_length = rep_len;
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
slot.off = repcode_to_offbase(rep_code - ll0 + 1);
slot.len = rep_len as u32;
match_count += 1;
if rep_len > sufficient_len as usize || ip.wrapping_add(rep_len) == input_limit {
return match_count;
}
}
}
if mls == 3 && best_length < mls as usize {
let match_index3 = unsafe { insert_and_find_first_index_hash3(state, next_to_update3, ip) };
if match_index3 >= match_low && current.wrapping_sub(match_index3) < (1 << 18) {
let match_ptr = if dict_mode == DICT_EXT && match_index3 < dict_limit {
state.dict_base.wrapping_add(match_index3 as usize)
} else {
state.base.wrapping_add(match_index3 as usize)
};
let match_length = if dict_mode == DICT_EXT && match_index3 < dict_limit {
unsafe { count_2segments(ip, match_ptr, input_limit, dict_end, prefix_start) }
} else {
unsafe { count(ip, match_ptr, input_limit) }
};
if match_length >= mls as usize {
let slot = unsafe { &mut *state.opt_match_table() };
slot.off = offset_to_offbase(current.wrapping_sub(match_index3));
slot.len = match_length as u32;
if match_length > sufficient_len as usize
|| ip.wrapping_add(match_length) == input_limit
{
unsafe { *state.next_to_update = current.wrapping_add(1) };
return 1;
}
match_count = 1;
best_length = match_length;
}
}
}
unsafe { table_set(state.hash_table, hash, current) };
while nb_compares != 0 && match_index >= match_low {
nb_compares -= 1;
let next_ptr = unsafe { state.chain_table.add(2 * (match_index & bt_mask) as usize) };
let mut match_length = min(common_smaller, common_larger);
let mut match_ptr: *const u8;
if dict_mode != DICT_EXT || match_index.wrapping_add(match_length as u32) >= dict_limit {
match_ptr = state.base.wrapping_add(match_index as usize);
match_length += unsafe {
count(
ip.wrapping_add(match_length),
match_ptr.wrapping_add(match_length),
input_limit,
)
};
} else {
match_ptr = state.dict_base.wrapping_add(match_index as usize);
match_length += unsafe {
count_2segments(
ip.wrapping_add(match_length),
match_ptr.wrapping_add(match_length),
input_limit,
dict_end,
prefix_start,
)
};
if match_index.wrapping_add(match_length as u32) >= dict_limit {
match_ptr = state.base.wrapping_add(match_index as usize);
}
}
if match_length > best_length {
if match_length > match_end_index.wrapping_sub(match_index) as usize {
match_end_index = match_index.wrapping_add(match_length as u32);
}
best_length = match_length;
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
slot.off = offset_to_offbase(current.wrapping_sub(match_index));
slot.len = match_length as u32;
match_count += 1;
if match_length > ZSTD_OPT_NUM as usize || ip.wrapping_add(match_length) == input_limit
{
if dict_mode == DICT_MATCH_STATE {
nb_compares = 0;
}
break;
}
}
let next_byte = unsafe { *match_ptr.add(match_length) };
let input_byte = unsafe { *ip.add(match_length) };
if next_byte < input_byte {
unsafe { *smaller = match_index };
common_smaller = match_length;
if match_index <= bt_low {
smaller = &mut dummy;
break;
}
smaller = unsafe { next_ptr.add(1) };
match_index = unsafe { *next_ptr.add(1) };
} else {
unsafe { *larger = match_index };
common_larger = match_length;
if match_index <= bt_low {
larger = &mut dummy;
break;
}
larger = next_ptr;
match_index = unsafe { *next_ptr };
}
}
unsafe {
*smaller = 0;
*larger = 0;
}
if dict_mode == DICT_MATCH_STATE && nb_compares != 0 {
let dict_hash = unsafe { hash_ptr(ip, dms_hash_log, mls) };
let mut dict_match_index = unsafe { table_get(dms.hash_table, dict_hash) };
common_smaller = 0;
common_larger = 0;
while nb_compares != 0 && dict_match_index > dms_low_limit {
nb_compares -= 1;
let next_ptr = unsafe {
dms.chain_table
.add(2 * (dict_match_index & dms_bt_mask) as usize)
};
let mut match_length = min(common_smaller, common_larger);
let mut match_ptr = dms_base.wrapping_add(dict_match_index as usize);
match_length += unsafe {
count_2segments(
ip.wrapping_add(match_length),
match_ptr.wrapping_add(match_length),
input_limit,
dms_end,
prefix_start,
)
};
if dict_match_index.wrapping_add(match_length as u32) >= dms_high_limit {
match_ptr = state
.base
.wrapping_add(dict_match_index.wrapping_add(dms_index_delta) as usize);
}
if match_length > best_length {
let prefix_index = dict_match_index.wrapping_add(dms_index_delta);
if match_length > match_end_index.wrapping_sub(prefix_index) as usize {
match_end_index = prefix_index.wrapping_add(match_length as u32);
}
best_length = match_length;
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
slot.off = offset_to_offbase(current.wrapping_sub(prefix_index));
slot.len = match_length as u32;
match_count += 1;
if match_length > ZSTD_OPT_NUM as usize
|| ip.wrapping_add(match_length) == input_limit
{
break;
}
}
if dict_match_index <= dms_bt_low {
break;
}
if unsafe { *match_ptr.add(match_length) } < unsafe { *ip.add(match_length) } {
common_smaller = match_length;
dict_match_index = unsafe { *next_ptr.add(1) };
} else {
common_larger = match_length;
dict_match_index = unsafe { *next_ptr };
}
}
}
unsafe { *state.next_to_update = match_end_index.wrapping_sub(8) };
match_count
}
impl ZSTD_RustOptState {
#[inline]
unsafe fn opt_match_table(&self) -> *mut ZSTD_match_t {
unsafe { (*self.opt).matchTable }
}
}
unsafe fn get_all_matches(
state: &mut ZSTD_RustOptState,
next_to_update3: &mut u32,
ip: *const u8,
input_limit: *const u8,
rep: &[u32; ZSTD_REP_NUM],
ll0: u32,
length_to_beat: u32,
dict_mode: c_int,
mls: u32,
) -> u32 {
debug_assert!((3..=6).contains(&mls));
if (ip as usize)
< state
.base
.wrapping_add(unsafe { *state.next_to_update } as usize) as usize
{
return 0;
}
unsafe {
ZSTD_rust_opt_updateTreeInternal(
(state as *mut ZSTD_RustOptState).cast::<OptTreePrefix>(),
ip.cast::<c_void>(),
input_limit.cast::<c_void>(),
mls,
c_int::from(dict_mode == DICT_EXT),
);
insert_bt_and_get_all_matches(
state,
next_to_update3,
ip,
input_limit,
dict_mode,
rep,
ll0,
length_to_beat,
mls,
)
}
}
unsafe fn resolve_shortest_path(
state: &mut ZSTD_RustOptState,
seq_store: *mut SeqStore_t,
rep: &mut [u32; ZSTD_REP_NUM],
istart: *const u8,
iend: *const u8,
anchor: &mut *const u8,
ip: &mut *const u8,
mut cur: u32,
last_pos: u32,
last_stretch: ZSTD_optimal_t,
opt_level: c_int,
) {
let opt_state = unsafe { &mut *state.opt };
let opt = opt_state.priceTable;
if last_stretch.mlen == 0 {
*ip = ip.wrapping_add(last_pos as usize);
return;
}
if last_stretch.litlen == 0 {
let previous = unsafe { (*opt.add(cur as usize)).rep };
*rep = new_rep(previous, last_stretch.off, unsafe {
(*opt.add(cur as usize)).litlen == 0
});
} else {
*rep = last_stretch.rep;
cur = cur.wrapping_sub(last_stretch.litlen);
}
let store_end = cur + 2;
let mut store_start = store_end;
let mut stretch_pos = cur;
unsafe {
*opt.add(store_end as usize) = last_stretch;
}
loop {
let next_stretch = unsafe { *opt.add(stretch_pos as usize) };
unsafe { (*opt.add(store_start as usize)).litlen = next_stretch.litlen };
if next_stretch.mlen == 0 {
break;
}
store_start -= 1;
unsafe { *opt.add(store_start as usize) = next_stretch };
stretch_pos = stretch_pos.wrapping_sub(next_stretch.litlen + next_stretch.mlen);
}
let mut store_pos = store_start;
while store_pos <= store_end {
let current = unsafe { *opt.add(store_pos as usize) };
let lit_length = current.litlen;
let match_length = current.mlen;
if match_length == 0 {
*ip = anchor.wrapping_add(lit_length as usize);
break;
}
unsafe {
update_stats(opt_state, lit_length, *anchor, current.off, match_length);
store_seq(
seq_store,
lit_length as usize,
*anchor,
current.off,
match_length as usize,
);
}
*anchor = anchor.wrapping_add((lit_length + match_length) as usize);
*ip = *anchor;
store_pos += 1;
}
unsafe { set_base_prices(opt_state, opt_level) };
let _ = istart;
let _ = iend;
}
unsafe fn compress_block_opt_generic(
state: &mut ZSTD_RustOptState,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
opt_level: c_int,
dict_mode: c_int,
) -> usize {
let istart = src;
let mut ip = istart;
let mut anchor = istart;
let iend = src.wrapping_add(src_size);
let ilimit = if src_size >= HASH_READ_SIZE {
iend.wrapping_sub(HASH_READ_SIZE)
} else {
istart
};
let prefix_start = state.base.wrapping_add(state.dict_limit as usize);
let sufficient_len = min(state.target_length, ZSTD_OPT_NUM - 1);
let min_match = if state.min_match == 3 { 3 } else { 4 };
let mut next_to_update3 = unsafe { *state.next_to_update };
let opt_state = unsafe { &mut *state.opt };
let matches = opt_state.matchTable;
let opt = opt_state.priceTable;
let mut last_stretch = ZSTD_optimal_t::default();
let mut opt_ldm = OptLdm {
seq_store: if state.ldm_seq_store.is_null() {
RawSeqStore::default()
} else {
unsafe { *state.ldm_seq_store }
},
start_pos: 0,
end_pos: 0,
offset: 0,
};
unsafe { ldm_next_match(&mut opt_ldm, 0, src_size as u32) };
unsafe { rescale_freqs(state, src, src_size, opt_level) };
if ip == prefix_start {
ip = ip.wrapping_add(1);
}
while (ip as usize) < ilimit as usize {
let mut last_pos;
let mut found_immediate = false;
let litlen = ptr_diff(ip, anchor) as u32;
let ll0 = u32::from(litlen == 0);
let mut nb_matches = unsafe {
get_all_matches(
state,
&mut next_to_update3,
ip,
iend,
&[*reps, *reps.add(1), *reps.add(2)],
ll0,
min_match,
dict_mode,
min_match,
)
};
unsafe {
ldm_process_match(
&mut opt_ldm,
matches,
&mut nb_matches,
ptr_diff(ip, istart) as u32,
ptr_diff(iend, ip) as u32,
min_match,
)
};
if nb_matches == 0 {
ip = ip.wrapping_add(1);
continue;
}
unsafe {
(*opt).mlen = 0;
(*opt).litlen = litlen;
(*opt).price = lit_length_price(litlen, opt_state, opt_level) as c_int;
(*opt).rep = [*reps, *reps.add(1), *reps.add(2)];
}
let max_match = unsafe { *matches.add(nb_matches as usize - 1) };
if max_match.len > sufficient_len {
last_stretch.litlen = 0;
last_stretch.mlen = max_match.len;
last_stretch.off = max_match.off;
last_pos = max_match.len;
found_immediate = true;
} else {
let mut pos = 1u32;
while pos < min_match {
unsafe {
(*opt.add(pos as usize)).price = ZSTD_MAX_PRICE;
(*opt.add(pos as usize)).mlen = 0;
(*opt.add(pos as usize)).litlen = litlen + pos;
}
pos += 1;
}
for match_index in 0..nb_matches as usize {
let candidate = unsafe { *matches.add(match_index) };
while pos <= candidate.len {
let match_price =
unsafe { get_match_price(candidate.off, pos, opt_state, opt_level) }
as c_int;
let sequence_price = unsafe { (*opt).price } + match_price;
unsafe {
(*opt.add(pos as usize)).mlen = pos;
(*opt.add(pos as usize)).off = candidate.off;
(*opt.add(pos as usize)).litlen = 0;
(*opt.add(pos as usize)).price =
sequence_price + lit_length_price(0, opt_state, opt_level) as c_int;
}
pos += 1;
}
}
last_pos = pos - 1;
unsafe { (*opt.add(pos as usize)).price = ZSTD_MAX_PRICE };
}
if found_immediate {
let mut next_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
unsafe {
resolve_shortest_path(
state,
seq_store,
&mut next_rep,
istart,
iend,
&mut anchor,
&mut ip,
0,
last_pos,
last_stretch,
opt_level,
)
};
unsafe {
*reps = next_rep[0];
*reps.add(1) = next_rep[1];
*reps.add(2) = next_rep[2];
}
continue;
}
let mut cur = 1u32;
let mut resolved_early = false;
while cur <= last_pos {
let inr = ip.wrapping_add(cur as usize);
let literal_length = unsafe { (*opt.add((cur - 1) as usize)).litlen } + 1;
let price = unsafe { (*opt.add((cur - 1) as usize)).price }
+ unsafe {
raw_literals_cost(ip.wrapping_add((cur - 1) as usize), 1, opt_state, opt_level)
} as c_int
+ (unsafe { lit_length_price(literal_length, opt_state, opt_level) } as c_int
- unsafe { lit_length_price(literal_length - 1, opt_state, opt_level) }
as c_int);
if price <= unsafe { (*opt.add(cur as usize)).price } {
let previous_match = unsafe { *opt.add(cur as usize) };
let previous = unsafe { *opt.add((cur - 1) as usize) };
unsafe { *opt.add(cur as usize) = previous };
unsafe {
(*opt.add(cur as usize)).litlen = literal_length;
(*opt.add(cur as usize)).price = price;
}
if opt_level >= 1
&& previous_match.litlen == 0
&& (unsafe { lit_length_price(1, opt_state, opt_level) } as c_int
- unsafe { lit_length_price(0, opt_state, opt_level) } as c_int)
< 0
&& (inr as usize) < (iend as usize)
{
let with_one = previous_match.price
+ unsafe { raw_literals_cost(inr, 1, opt_state, opt_level) } as c_int
+ (unsafe { lit_length_price(1, opt_state, opt_level) } as c_int
- unsafe { lit_length_price(0, opt_state, opt_level) } as c_int);
let with_more = price
+ unsafe { raw_literals_cost(inr, 1, opt_state, opt_level) } as c_int
+ (unsafe { lit_length_price(literal_length + 1, opt_state, opt_level) }
as c_int
- unsafe { lit_length_price(literal_length, opt_state, opt_level) }
as c_int);
if with_one < with_more
&& with_one < unsafe { (*opt.add((cur + 1) as usize)).price }
{
let previous_index = cur - previous_match.mlen;
let reps = new_rep(
unsafe { (*opt.add(previous_index as usize)).rep },
previous_match.off,
unsafe { (*opt.add(previous_index as usize)).litlen == 0 },
);
unsafe {
(*opt.add((cur + 1) as usize)).rep = reps;
(*opt.add((cur + 1) as usize)).mlen = previous_match.mlen;
(*opt.add((cur + 1) as usize)).off = previous_match.off;
(*opt.add((cur + 1) as usize)).litlen = 1;
(*opt.add((cur + 1) as usize)).price = with_one;
}
if last_pos < cur + 1 {
last_pos = cur + 1;
}
}
}
}
let selected = unsafe { *opt.add(cur as usize) };
if selected.litlen == 0 {
let previous_index = cur - selected.mlen;
let reps = new_rep(
unsafe { (*opt.add(previous_index as usize)).rep },
selected.off,
unsafe { (*opt.add(previous_index as usize)).litlen == 0 },
);
unsafe { (*opt.add(cur as usize)).rep = reps };
}
if (inr as usize) > (ilimit as usize) {
cur += 1;
continue;
}
if cur == last_pos {
break;
}
if opt_level == 0
&& unsafe { (*opt.add((cur + 1) as usize)).price }
<= unsafe { (*opt.add(cur as usize)).price } + (BITCOST_MULTIPLIER / 2) as c_int
{
cur += 1;
continue;
}
let previous_price = unsafe { (*opt.add(cur as usize)).price };
let base_price =
previous_price + unsafe { lit_length_price(0, opt_state, opt_level) } as c_int;
let current_rep = unsafe { (*opt.add(cur as usize)).rep };
let mut candidates = unsafe {
get_all_matches(
state,
&mut next_to_update3,
inr,
iend,
&current_rep,
u32::from(selected.litlen == 0),
min_match,
dict_mode,
min_match,
)
};
unsafe {
ldm_process_match(
&mut opt_ldm,
matches,
&mut candidates,
ptr_diff(inr, istart) as u32,
ptr_diff(iend, inr) as u32,
min_match,
)
};
if candidates == 0 {
cur += 1;
continue;
}
let longest = unsafe { (*matches.add(candidates as usize - 1)).len };
if longest > sufficient_len
|| cur + longest >= ZSTD_OPT_NUM
|| ptr_diff(iend, inr) <= longest as usize
{
last_stretch.mlen = longest;
last_stretch.off = unsafe { (*matches.add(candidates as usize - 1)).off };
last_stretch.litlen = 0;
last_pos = cur + longest;
let mut path_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
unsafe {
resolve_shortest_path(
state,
seq_store,
&mut path_rep,
istart,
iend,
&mut anchor,
&mut ip,
cur,
last_pos,
last_stretch,
opt_level,
)
};
unsafe {
*reps = path_rep[0];
*reps.add(1) = path_rep[1];
*reps.add(2) = path_rep[2];
}
resolved_early = true;
break;
}
for match_index in 0..candidates as usize {
let candidate = unsafe { *matches.add(match_index) };
let start_ml = if match_index > 0 {
unsafe { (*matches.add(match_index - 1)).len + 1 }
} else {
min_match
};
let mut match_length = candidate.len;
loop {
let position = cur + match_length;
let match_price = unsafe {
get_match_price(candidate.off, match_length, opt_state, opt_level)
} as c_int;
let candidate_price = base_price + match_price;
if position > last_pos
|| candidate_price < unsafe { (*opt.add(position as usize)).price }
{
while last_pos < position {
last_pos += 1;
unsafe {
(*opt.add(last_pos as usize)).price = ZSTD_MAX_PRICE;
(*opt.add(last_pos as usize)).litlen = 1;
}
}
unsafe {
(*opt.add(position as usize)).mlen = match_length;
(*opt.add(position as usize)).off = candidate.off;
(*opt.add(position as usize)).litlen = 0;
(*opt.add(position as usize)).price = candidate_price;
}
} else if opt_level == 0 {
break;
}
if match_length == start_ml {
break;
}
match_length -= 1;
}
}
unsafe { (*opt.add((last_pos + 1) as usize)).price = ZSTD_MAX_PRICE };
cur += 1;
}
if !resolved_early {
/* The previous loop reached its end without an early path. */
last_stretch = unsafe { *opt.add(last_pos as usize) };
let path_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
let mut next_rep = path_rep;
let path_cur = last_pos.wrapping_sub(last_stretch.mlen);
unsafe {
resolve_shortest_path(
state,
seq_store,
&mut next_rep,
istart,
iend,
&mut anchor,
&mut ip,
path_cur,
last_pos,
last_stretch,
opt_level,
)
};
unsafe {
*reps = next_rep[0];
*reps.add(1) = next_rep[1];
*reps.add(2) = next_rep[2];
}
}
}
ptr_diff(iend, anchor)
}
unsafe fn init_stats_ultra(
state: &mut ZSTD_RustOptState,
seq_store: *mut SeqStore_t,
reps: *const u32,
src: *const u8,
src_size: usize,
) {
let mut temporary_rep = [unsafe { *reps }, unsafe { *reps.add(1) }, unsafe {
*reps.add(2)
}];
unsafe {
compress_block_opt_generic(
state,
seq_store,
temporary_rep.as_mut_ptr(),
src,
src_size,
2,
DICT_NO_DICT,
);
}
let store = unsafe { &mut *seq_store };
store.sequences = store.sequencesStart;
store.lit = store.litStart;
store.longLengthType = 0;
unsafe {
*state.window_base = (*state.window_base).wrapping_sub(src_size);
*state.window_dict_limit = (*state.window_dict_limit).wrapping_add(src_size as u32);
*state.window_low_limit = *state.window_dict_limit;
*state.next_to_update = *state.window_dict_limit;
refresh_window_projection(state);
}
}
unsafe fn refresh_window_projection(state: &mut ZSTD_RustOptState) {
/* The C implementation rebuilds its projected match state for the
* second pass. Refresh the copied window fields before reusing this
* Rust projection for that pass. */
state.base = unsafe { *state.window_base };
state.dict_limit = unsafe { *state.window_dict_limit };
state.low_limit = unsafe { *state.window_low_limit };
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_opt_updateTree(
state: *mut ZSTD_RustOptState,
ip: *const c_void,
iend: *const c_void,
mls: u32,
dict_mode: c_int,
) {
let state = unsafe { &mut *state };
unsafe {
ZSTD_rust_opt_updateTreeInternal(
(state as *mut ZSTD_RustOptState).cast::<OptTreePrefix>(),
ip,
iend,
mls,
c_int::from(dict_mode == DICT_EXT),
)
};
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_opt(
state: *mut ZSTD_RustOptState,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
opt_level: c_int,
dict_mode: c_int,
) -> usize {
let state = unsafe { &mut *state };
unsafe {
compress_block_opt_generic(
state,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
opt_level,
dict_mode,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_initStats_ultra(
state: *mut ZSTD_RustOptState,
seq_store: *mut c_void,
reps: *const u32,
src: *const c_void,
src_size: usize,
) {
let state = unsafe { &mut *state };
unsafe {
init_stats_ultra(
state,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
)
};
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_btultra2(
state: *mut ZSTD_RustOptState,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
) -> usize {
let state = unsafe { &mut *state };
let store = unsafe { &*seq_store.cast::<SeqStore_t>() };
let current = index_from(state.base, src.cast::<u8>());
let should_seed = unsafe { (*state.opt).litLengthSum == 0 }
&& store.sequences == store.sequencesStart
&& unsafe { *state.window_dict_limit == *state.window_low_limit }
&& current == unsafe { *state.window_dict_limit }
&& src_size > ZSTD_PREDEF_THRESHOLD;
if should_seed {
unsafe { init_stats_ultra(state, seq_store.cast(), reps, src.cast(), src_size) };
}
unsafe {
compress_block_opt_generic(
state,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
2,
DICT_NO_DICT,
)
}
}
#[cfg(test)]
mod tests {
use super::{
downscale_stats, index_overlap_check, refresh_window_projection, ZSTD_RustOptState,
};
use std::ptr;
#[test]
fn index_overlap_check_accepts_exact_non_overlapping_boundary() {
assert!(index_overlap_check(16, 12, 16));
assert!(!index_overlap_check(16, 13, 16));
assert!(index_overlap_check(16, 16, 16));
}
#[test]
fn index_overlap_check_rejects_wrapped_indices() {
assert!(!index_overlap_check(0, u32::MAX - 3, 0x1fb));
assert!(!index_overlap_check(3, u32::MAX, 2));
assert!(index_overlap_check(u32::MAX, u32::MAX, u32::MAX));
}
#[test]
fn ultra2_window_projection_tracks_the_second_pass_window() {
let mut base = 0x2000usize as *const u8;
let mut dict_limit = 128_u32;
let mut low_limit = 128_u32;
let mut next_to_update = 128_u32;
let mut state = ZSTD_RustOptState {
hash_table: ptr::null_mut(),
chain_table: ptr::null_mut(),
base: 0x1000usize as *const u8,
dict_base: ptr::null(),
dict_limit: 0,
low_limit: 0,
loaded_dict_end: 0,
next_to_update: &mut next_to_update,
hash_log: 0,
chain_log: 0,
search_log: 0,
window_log: 0,
use_c_predict: 0,
hash_table3: ptr::null_mut(),
hash_log3: 0,
next_src: ptr::null(),
min_match: 0,
target_length: 0,
opt: ptr::null_mut(),
dict_match_state: ptr::null(),
ldm_seq_store: ptr::null(),
window_base: &mut base,
window_dict_limit: &mut dict_limit,
window_low_limit: &mut low_limit,
huf_ctable: ptr::null(),
huf_repeat_valid: 0,
fse_litlength_ctable: ptr::null(),
fse_matchlength_ctable: ptr::null(),
fse_offcode_ctable: ptr::null(),
};
unsafe { refresh_window_projection(&mut state) };
assert_eq!(state.base, base);
assert_eq!(state.dict_limit, dict_limit);
assert_eq!(state.low_limit, low_limit);
}
#[test]
fn downscale_stats_preserves_observed_symbols_with_base_zero_possible() {
let mut frequencies = [0_u32, 1, 255, 256, 511];
let sum = unsafe { downscale_stats(frequencies.as_mut_ptr(), 4, 8, false) };
assert_eq!(frequencies, [0, 1, 1, 2, 2]);
assert_eq!(sum, 6);
}
}
Reference in New Issue View Git Blame Copy Permalink