diff --git a/rust/src/dict_builder_zdict.rs b/rust/src/dict_builder_zdict.rs index 5488b2fed..0bb1e0b6c 100644 --- a/rust/src/dict_builder_zdict.rs +++ b/rust/src/dict_builder_zdict.rs @@ -8,9 +8,9 @@ //! This is the Rust translation of `lib/dictBuilder/zdict.c`. The COVER and //! fastCOVER implementations remain separate translation units, but their //! finalization calls, the legacy trainer, entropy-header construction, and -//! public helper functions all live here. Compression contexts remain opaque: -//! the statistics pass uses the existing C context API and only projects the -//! stable `SeqStore_t` leaf into Rust. +//! public helper functions all live here. The entropy statistics pass owns a +//! private Rust context and passes only field-level projections to the Rust +//! matcher leaves. use crate::bits::ZSTD_highbit32; use crate::common::{LL_FSE_LOG, MAX_LL, MAX_ML, ML_FSE_LOG, OFF_FSE_LOG, ZSTD_REP_NUM}; @@ -24,11 +24,18 @@ use crate::huf_compress::{HUF_buildCTable_wksp, HUF_writeCTable_wksp}; use crate::mem::{MEM_readLE32, MEM_writeLE32}; use crate::xxhash::XXH64; use crate::zstd_compress_dictionary::ZSTD_rust_loadCEntropy; -use crate::zstd_compress_params::{ZSTD_compressionParameters, ZSTD_parameters}; +use crate::zstd_compress_params::{ + ZSTD_compressionParameters, ZSTD_parameters, ZSTD_rust_params_adjustCParams, + ZSTD_rust_params_allocateChainTable, ZSTD_rust_params_getCParamsFromCCtxParams, + ZSTD_rust_params_makeParams, ZSTD_rust_params_maxNbSeq, + ZSTD_rust_params_resolveRowMatchFinderMode, ZSTD_rust_params_selectCParams, + ZSTD_RUST_CPM_CREATE_CDICT, ZSTD_RUST_CPM_UNKNOWN, ZSTD_RUST_PS_AUTO, ZSTD_RUST_PS_ENABLE, +}; use crate::zstd_compress_sequences::SeqDef; use crate::zstd_compress_stats::{ SeqStore_t, ZSTD_compressedBlockState_t, ZSTD_rust_resetCompressedBlockState, ZSTD_seqToCodes, }; +use crate::zstd_compress_superblock::ZSTD_rust_compressSuperBlock; use std::ffi::{c_char, c_void}; use std::mem::{size_of, MaybeUninit}; use std::os::raw::{c_int, c_uint}; @@ -51,12 +58,6 @@ const MINMATCHLENGTH: usize = 7; const MAXREPOFFSET: usize = 1024; const OFFCODE_MAX: usize = 30; -const ZSTD_DLM_BY_REF: c_int = 1; -const ZSTD_DCT_RAW_CONTENT: c_int = 1; - -type ZstdAllocFunction = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void; -type ZstdFreeFunction = unsafe extern "C" fn(*mut c_void, *mut c_void); - /// ABI-compatible `ZDICT_params_t` from `zdict.h`. #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -74,54 +75,6 @@ pub struct ZDICT_legacy_params_t { pub zParams: ZDICT_params_t, } -type ZSTD_CCtx = c_void; -type ZSTD_CDict = c_void; - -#[repr(C)] -#[derive(Clone, Copy)] -struct ZSTD_customMem { - customAlloc: Option, - customFree: Option, - opaque: *mut c_void, -} - -const ZSTD_DEFAULT_CMEM: ZSTD_customMem = ZSTD_customMem { - customAlloc: None, - customFree: None, - opaque: ptr::null_mut(), -}; - -unsafe extern "C" { - fn ZSTD_getParams( - compression_level: c_int, - estimated_src_size: u64, - dict_size: usize, - ) -> ZSTD_parameters; - fn ZSTD_createCDict_advanced( - dict: *const c_void, - dict_size: usize, - dict_load_method: c_int, - dict_content_type: c_int, - c_params: ZSTD_compressionParameters, - custom_mem: ZSTD_customMem, - ) -> *mut ZSTD_CDict; - fn ZSTD_freeCDict(cdict: *mut ZSTD_CDict) -> usize; - fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - fn ZSTD_freeCCtx(cctx: *mut ZSTD_CCtx) -> usize; - fn ZSTD_compressBegin_usingCDict_deprecated( - cctx: *mut ZSTD_CCtx, - cdict: *const ZSTD_CDict, - ) -> usize; - fn ZSTD_compressBlock_deprecated( - cctx: *mut ZSTD_CCtx, - dst: *mut c_void, - dst_capacity: usize, - src: *const c_void, - src_size: usize, - ) -> usize; - fn ZSTD_getSeqStore(cctx: *const ZSTD_CCtx) -> *const SeqStore_t; -} - #[inline] fn dictionary_error(code: ZstdErrorCode) -> usize { ERROR(code) @@ -538,9 +491,815 @@ fn train_buffer_legacy( 0 } +const ZSTD_FAST: c_int = 1; +const ZSTD_DFAST: c_int = 2; +const ZSTD_GREEDY: c_int = 3; +const ZSTD_LAZY: c_int = 4; +const ZSTD_LAZY2: c_int = 5; +const ZSTD_BTLAZY2: c_int = 6; +const ZSTD_BTOPT: c_int = 7; +const ZSTD_BTULTRA: c_int = 8; +const ZSTD_BTULTRA2: c_int = 9; +const ZSTD_DICT_NO_DICT: c_int = 0; +const ZSTD_DICT_MATCH_STATE: c_int = 2; +const ZSTD_SEARCH_HASH_CHAIN: c_int = 0; +const ZSTD_SEARCH_BINARY_TREE: c_int = 1; +const ZSTD_SEARCH_ROW_HASH: c_int = 2; +const HASH_READ_SIZE: usize = 8; +const HASHLOG3_MAX: u32 = 17; +const ZSTD_OPT_SIZE: usize = (1 << 12) + 3; +const TMP_WORKSPACE_SIZE: usize = 16 << 10; +const DICT_WINDOW_START_INDEX: u32 = 2; + +/// This is the same field-level projection used by `zstd_lazy.c`. +/// +/// It is deliberately not a projection of `ZSTD_CCtx_s` or +/// `ZSTD_MatchState_t`; the Rust matcher owns this private value and only its +/// already-established leaf ABI crosses into the matcher module. +#[repr(C)] +struct DictLazyStateProjection { + hashTable: *mut u32, + chainTable: *mut u32, + tagTable: *mut u8, + hashCache: *mut u32, + base: *const u8, + dictBase: *const u8, + nextSrc: *const u8, + dictLimit: u32, + lowLimit: u32, + loadedDictEnd: u32, + nextToUpdate: *mut u32, + lazySkipping: *mut c_int, + hashLog: u32, + chainLog: u32, + minMatch: u32, + searchLog: u32, + windowLog: u32, + rowHashLog: u32, + hashSalt: u64, + hashSaltEntropy: *mut u32, + dictMatchState: *const DictLazyStateProjection, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct DictMatchProjection { + off: u32, + len: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct DictOptimalProjection { + price: c_int, + off: u32, + mlen: u32, + litlen: u32, + rep: [u32; ZSTD_REP_NUM], +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct DictRawSeqProjection { + offset: u32, + litLength: u32, + matchLength: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct DictRawSeqStoreProjection { + seq: *const DictRawSeqProjection, + pos: usize, + posInSequence: usize, + size: usize, + capacity: usize, +} + +#[repr(C)] +struct DictOptStateProjection { + litFreq: *mut c_uint, + litLengthFreq: *mut c_uint, + matchLengthFreq: *mut c_uint, + offCodeFreq: *mut c_uint, + matchTable: *mut DictMatchProjection, + priceTable: *mut DictOptimalProjection, + litSum: u32, + litLengthSum: u32, + matchLengthSum: u32, + offCodeSum: u32, + litSumBasePrice: u32, + litLengthSumBasePrice: u32, + matchLengthSumBasePrice: u32, + offCodeSumBasePrice: u32, + priceType: c_int, + symbolCosts: *const c_void, + literalCompressionMode: c_int, +} + +/// This is the field-level projection used by `zstd_opt.c` and +/// `zstd_opt_tree.c`. The private C match-state itself never crosses here. +#[repr(C)] +struct DictOptStateProjectionView { + hash_table: *mut u32, + chain_table: *mut u32, + base: *const u8, + dict_base: *const u8, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + next_to_update: *mut u32, + hash_log: u32, + chain_log: u32, + search_log: u32, + window_log: u32, + use_c_predict: c_int, + hash_table3: *mut u32, + hash_log3: u32, + next_src: *const u8, + min_match: u32, + target_length: u32, + opt: *mut DictOptStateProjection, + dict_match_state: *const DictOptStateProjectionView, + ldm_seq_store: *const DictRawSeqStoreProjection, + window_base: *mut *const u8, + window_dict_limit: *mut u32, + window_low_limit: *mut u32, + huf_ctable: *const usize, + huf_repeat_valid: c_int, + fse_litlength_ctable: *const u32, + fse_matchlength_ctable: *const u32, + fse_offcode_ctable: *const u32, +} + +fn try_zeroed_vec(length: usize) -> Result, ()> { + let mut result = Vec::new(); + result.try_reserve_exact(length).map_err(|_| ())?; + result.resize(length, T::default()); + Ok(result) +} + +fn table_length(log: u32, entries_per_slot: usize) -> Result { + 1usize + .checked_shl(log) + .and_then(|slots| slots.checked_mul(entries_per_slot)) + .ok_or(()) +} + +#[allow(dead_code)] +struct DictEntropyContext { + cparams: ZSTD_compressionParameters, + row_match_finder: bool, + has_dictionary: bool, + dict_storage: Vec, + dict_hash_table: Vec, + dict_chain_table: Vec, + dict_tag_table: Vec, + active_hash_table: Vec, + active_chain_table: Vec, + active_hash_table3: Vec, + active_tag_table: Vec, + active_hash_cache: Vec, + dict_next_to_update: u32, + dict_lazy_skipping: c_int, + sequences: Vec, + literals: Vec, + ll_codes: Vec, + ml_codes: Vec, + of_codes: Vec, + seq_store: SeqStore_t, + opt_lit_freq: Vec, + opt_lit_length_freq: Vec, + opt_match_length_freq: Vec, + opt_off_code_freq: Vec, + opt_match_table: Vec, + opt_price_table: Vec, + active_opt_state: DictOptStateProjection, +} + +impl DictEntropyContext { + fn new( + cparams: ZSTD_compressionParameters, + dict_buffer: *const u8, + dict_size: usize, + ) -> Result { + let row_match_finder = + ZSTD_rust_params_resolveRowMatchFinderMode(ZSTD_RUST_PS_AUTO, cparams) + == ZSTD_RUST_PS_ENABLE; + let hash_length = table_length(cparams.hashLog, 1)?; + let chain_length = if ZSTD_rust_params_allocateChainTable( + cparams.strategy, + if row_match_finder { + ZSTD_RUST_PS_ENABLE + } else { + 2 + }, + 0, + ) != 0 + { + table_length(cparams.chainLog, 1)? + } else { + 0 + }; + let has_dictionary = dict_size > HASH_READ_SIZE; + let dictionary_length = dict_size + .checked_add(DICT_WINDOW_START_INDEX as usize) + .ok_or(())?; + let mut dict_storage: Vec = try_zeroed_vec(dictionary_length)?; + if dict_size != 0 { + if dict_buffer.is_null() { + return Err(()); + } + unsafe { + ptr::copy_nonoverlapping( + dict_buffer, + dict_storage + .as_mut_ptr() + .add(DICT_WINDOW_START_INDEX as usize), + dict_size, + ); + } + } + + let dict_hash_table = if has_dictionary { + try_zeroed_vec(hash_length)? + } else { + Vec::new() + }; + let dict_chain_table = if has_dictionary && chain_length != 0 { + try_zeroed_vec(chain_length)? + } else { + Vec::new() + }; + let dict_tag_table = if has_dictionary && row_match_finder { + try_zeroed_vec(hash_length)? + } else { + Vec::new() + }; + let active_hash_table = try_zeroed_vec(hash_length)?; + let active_chain_table = if chain_length != 0 { + try_zeroed_vec(chain_length)? + } else { + Vec::new() + }; + let active_hash_table3 = if cparams.strategy >= ZSTD_BTOPT && cparams.minMatch == 3 { + try_zeroed_vec(table_length(HASHLOG3_MAX.min(cparams.windowLog), 1)?)? + } else { + Vec::new() + }; + let active_tag_table = if row_match_finder { + try_zeroed_vec(hash_length)? + } else { + Vec::new() + }; + let active_hash_cache = if row_match_finder { + try_zeroed_vec(8)? + } else { + Vec::new() + }; + + let max_nb_seq = + ZSTD_rust_params_maxNbSeq(ZSTD_BLOCKSIZE_MAX, cparams.minMatch, 0).saturating_add(1); + let sequences: Vec = try_zeroed_vec(max_nb_seq)?; + let literals: Vec = try_zeroed_vec(ZSTD_BLOCKSIZE_MAX)?; + let ll_codes: Vec = try_zeroed_vec(max_nb_seq)?; + let ml_codes: Vec = try_zeroed_vec(max_nb_seq)?; + let of_codes: Vec = try_zeroed_vec(max_nb_seq)?; + let opt_lit_freq: Vec = try_zeroed_vec(256)?; + let opt_lit_length_freq: Vec = try_zeroed_vec(MAX_LL + 1)?; + let opt_match_length_freq: Vec = try_zeroed_vec(MAX_ML + 1)?; + let opt_off_code_freq: Vec = try_zeroed_vec(crate::common::MAX_OFF + 1)?; + let opt_match_table: Vec = try_zeroed_vec(ZSTD_OPT_SIZE)?; + let opt_price_table: Vec = try_zeroed_vec(ZSTD_OPT_SIZE)?; + + let active_opt_state = DictOptStateProjection { + litFreq: opt_lit_freq.as_ptr().cast_mut(), + litLengthFreq: opt_lit_length_freq.as_ptr().cast_mut(), + matchLengthFreq: opt_match_length_freq.as_ptr().cast_mut(), + offCodeFreq: opt_off_code_freq.as_ptr().cast_mut(), + matchTable: opt_match_table.as_ptr().cast_mut(), + priceTable: opt_price_table.as_ptr().cast_mut(), + litSum: 0, + litLengthSum: 0, + matchLengthSum: 0, + offCodeSum: 0, + litSumBasePrice: 0, + litLengthSumBasePrice: 0, + matchLengthSumBasePrice: 0, + offCodeSumBasePrice: 0, + priceType: 0, + symbolCosts: ptr::null(), + literalCompressionMode: 0, + }; + let seq_store = SeqStore_t { + sequencesStart: sequences.as_ptr().cast_mut(), + sequences: sequences.as_ptr().cast_mut(), + litStart: literals.as_ptr().cast_mut(), + lit: literals.as_ptr().cast_mut(), + llCode: ll_codes.as_ptr().cast_mut(), + mlCode: ml_codes.as_ptr().cast_mut(), + ofCode: of_codes.as_ptr().cast_mut(), + maxNbSeq: max_nb_seq, + maxNbLit: ZSTD_BLOCKSIZE_MAX, + longLengthType: 0, + longLengthPos: 0, + }; + let mut context = Self { + cparams, + row_match_finder, + has_dictionary, + dict_storage, + dict_hash_table, + dict_chain_table, + dict_tag_table, + active_hash_table, + active_chain_table, + active_hash_table3, + active_tag_table, + active_hash_cache, + dict_next_to_update: DICT_WINDOW_START_INDEX, + dict_lazy_skipping: 0, + sequences, + literals, + ll_codes, + ml_codes, + of_codes, + seq_store, + opt_lit_freq, + opt_lit_length_freq, + opt_match_length_freq, + opt_off_code_freq, + opt_match_table, + opt_price_table, + active_opt_state, + }; + context.load_dictionary_tables(); + Ok(context) + } + + fn load_dictionary_tables(&mut self) { + if !self.has_dictionary { + return; + } + let dict_base = self.dict_storage.as_ptr(); + let dict_end = unsafe { dict_base.add(self.dict_storage.len()) }; + let strategy = self.cparams.strategy; + let min_match = self.cparams.minMatch; + let hash_log = self.cparams.hashLog; + let chain_log = self.cparams.chainLog; + unsafe { + match strategy { + ZSTD_FAST => crate::zstd_fast::ZSTD_rust_fillHashTable( + self.dict_hash_table.as_mut_ptr(), + dict_base, + DICT_WINDOW_START_INDEX, + dict_end.cast(), + hash_log, + min_match, + 1, + 1, + ), + ZSTD_DFAST => crate::zstd_double_fast::ZSTD_rust_fillDoubleHashTable( + self.dict_hash_table.as_mut_ptr(), + self.dict_chain_table.as_mut_ptr(), + dict_base, + DICT_WINDOW_START_INDEX, + dict_end.cast(), + hash_log, + chain_log, + min_match, + 1, + 1, + ), + ZSTD_GREEDY | ZSTD_LAZY | ZSTD_LAZY2 => { + let row_log = self.cparams.searchLog.clamp(4, 6); + let mut state = DictLazyStateProjection { + hashTable: self.dict_hash_table.as_mut_ptr(), + chainTable: self.dict_chain_table.as_mut_ptr(), + tagTable: self.dict_tag_table.as_mut_ptr(), + hashCache: ptr::null_mut(), + base: dict_base, + dictBase: dict_base, + nextSrc: dict_end, + dictLimit: DICT_WINDOW_START_INDEX, + lowLimit: DICT_WINDOW_START_INDEX, + loadedDictEnd: self.dict_storage.len() as u32, + nextToUpdate: ptr::addr_of_mut!(self.dict_next_to_update), + lazySkipping: ptr::null_mut(), + hashLog: hash_log, + chainLog: chain_log, + minMatch: min_match, + searchLog: self.cparams.searchLog, + windowLog: self.cparams.windowLog, + rowHashLog: hash_log - row_log, + hashSalt: 0, + hashSaltEntropy: ptr::null_mut(), + dictMatchState: ptr::null(), + }; + if self.row_match_finder { + self.dict_tag_table.fill(0); + crate::zstd_lazy::ZSTD_rust_lazy_row_update( + (&mut state as *mut DictLazyStateProjection).cast(), + dict_end.sub(HASH_READ_SIZE).cast(), + ); + } else { + crate::zstd_lazy::ZSTD_rust_lazy_insertAndFindFirstIndex( + (&mut state as *mut DictLazyStateProjection).cast(), + dict_end.sub(HASH_READ_SIZE).cast(), + ); + } + } + ZSTD_BTLAZY2 | ZSTD_BTOPT | ZSTD_BTULTRA | ZSTD_BTULTRA2 => { + let mut state = Self::make_opt_projection( + self.cparams, + self.dict_hash_table.as_mut_ptr(), + self.dict_chain_table.as_mut_ptr(), + ptr::null_mut(), + ptr::addr_of_mut!(self.dict_next_to_update), + dict_base, + dict_base, + dict_end, + DICT_WINDOW_START_INDEX, + DICT_WINDOW_START_INDEX, + self.dict_storage.len() as u32, + ptr::null_mut(), + ptr::null(), + ); + crate::zstd_opt::ZSTD_rust_opt_updateTree( + (&mut state as *mut DictOptStateProjectionView).cast(), + dict_end.sub(HASH_READ_SIZE).cast(), + dict_end.cast(), + min_match, + ZSTD_DICT_NO_DICT, + ); + } + _ => {} + } + } + self.dict_next_to_update = self.dict_storage.len() as u32; + } + + fn reset_active_tables(&mut self) { + self.active_hash_table.fill(0); + self.active_chain_table.fill(0); + self.active_hash_table3.fill(0); + self.active_tag_table.fill(0); + self.active_hash_cache.fill(0); + self.dict_lazy_skipping = 0; + self.active_opt_state.litSum = 0; + self.active_opt_state.litLengthSum = 0; + self.active_opt_state.matchLengthSum = 0; + self.active_opt_state.offCodeSum = 0; + self.active_opt_state.litSumBasePrice = 0; + self.active_opt_state.litLengthSumBasePrice = 0; + self.active_opt_state.matchLengthSumBasePrice = 0; + self.active_opt_state.offCodeSumBasePrice = 0; + self.active_opt_state.priceType = 0; + self.opt_lit_freq.fill(0); + self.opt_lit_length_freq.fill(0); + self.opt_match_length_freq.fill(0); + self.opt_off_code_freq.fill(0); + self.opt_match_table.fill(DictMatchProjection::default()); + self.opt_price_table.fill(DictOptimalProjection::default()); + } + + #[allow(clippy::too_many_arguments)] + fn make_opt_projection( + cparams: ZSTD_compressionParameters, + hash_table: *mut u32, + chain_table: *mut u32, + hash_table3: *mut u32, + next_to_update: *mut u32, + base: *const u8, + dict_base: *const u8, + next_src: *const u8, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + opt: *mut DictOptStateProjection, + dict_match_state: *const DictOptStateProjectionView, + ) -> DictOptStateProjectionView { + DictOptStateProjectionView { + hash_table, + chain_table, + base, + dict_base, + dict_limit, + low_limit, + loaded_dict_end, + next_to_update, + hash_log: cparams.hashLog, + chain_log: cparams.chainLog, + search_log: cparams.searchLog, + window_log: cparams.windowLog, + use_c_predict: 0, + hash_table3, + hash_log3: HASHLOG3_MAX.min(cparams.windowLog), + next_src, + min_match: cparams.minMatch, + target_length: cparams.targetLength, + opt, + dict_match_state, + ldm_seq_store: ptr::null(), + window_base: ptr::null_mut(), + window_dict_limit: ptr::null_mut(), + window_low_limit: ptr::null_mut(), + huf_ctable: ptr::null(), + huf_repeat_valid: 0, + fse_litlength_ctable: ptr::null(), + fse_matchlength_ctable: ptr::null(), + fse_offcode_ctable: ptr::null(), + } + } + + #[allow(clippy::too_many_arguments)] + fn compress_sample( + &mut self, + workplace: &mut [u8], + params: &ZSTD_parameters, + src: *const c_void, + mut src_size: usize, + ) -> bool { + let block_size_max = ZSTD_BLOCKSIZE_MAX.min(1usize << params.cParams.windowLog); + src_size = src_size.min(block_size_max); + if src_size < HASH_READ_SIZE { + return false; + } + + self.reset_active_tables(); + self.seq_store.sequences = self.seq_store.sequencesStart; + self.seq_store.lit = self.seq_store.litStart; + self.seq_store.longLengthType = 0; + self.seq_store.longLengthPos = 0; + let source = src.cast::(); + let mut reps = [1u32, 4, 8]; + let dict_mode = if self.has_dictionary { + ZSTD_DICT_MATCH_STATE + } else { + ZSTD_DICT_NO_DICT + }; + let last_literals = unsafe { + match self.cparams.strategy { + ZSTD_FAST if self.has_dictionary => { + crate::zstd_fast::ZSTD_rust_compressBlock_fast_dictMatchState( + self.active_hash_table.as_mut_ptr(), + source, + 0, + self.cparams.hashLog, + self.cparams.minMatch, + self.cparams.targetLength, + (&mut self.seq_store as *mut SeqStore_t).cast(), + reps.as_mut_ptr(), + src, + src_size, + self.dict_hash_table.as_ptr(), + self.dict_storage.as_ptr(), + DICT_WINDOW_START_INDEX, + self.dict_storage.as_ptr().add(self.dict_storage.len()), + self.cparams.hashLog, + 0, + ) + } + ZSTD_FAST => crate::zstd_fast::ZSTD_rust_compressBlock_fast( + self.active_hash_table.as_mut_ptr(), + source, + 0, + 0, + self.cparams.hashLog, + self.cparams.minMatch, + self.cparams.targetLength, + self.cparams.windowLog, + (&mut self.seq_store as *mut SeqStore_t).cast(), + reps.as_mut_ptr(), + src, + src_size, + ), + ZSTD_DFAST if self.has_dictionary => { + crate::zstd_double_fast::ZSTD_rust_compressBlock_doubleFast_dictMatchState( + self.active_hash_table.as_mut_ptr(), + self.active_chain_table.as_mut_ptr(), + source, + 0, + self.cparams.hashLog, + self.cparams.chainLog, + self.cparams.minMatch, + (&mut self.seq_store as *mut SeqStore_t).cast(), + reps.as_mut_ptr(), + src, + src_size, + self.dict_hash_table.as_ptr(), + self.dict_chain_table.as_ptr(), + self.dict_storage.as_ptr(), + DICT_WINDOW_START_INDEX, + self.dict_storage.as_ptr().add(self.dict_storage.len()), + self.cparams.hashLog, + self.cparams.chainLog, + 0, + ) + } + ZSTD_DFAST => crate::zstd_double_fast::ZSTD_rust_compressBlock_doubleFast( + self.active_hash_table.as_mut_ptr(), + self.active_chain_table.as_mut_ptr(), + source, + 0, + 0, + self.cparams.hashLog, + self.cparams.chainLog, + self.cparams.minMatch, + self.cparams.windowLog, + (&mut self.seq_store as *mut SeqStore_t).cast(), + reps.as_mut_ptr(), + src, + src_size, + ), + ZSTD_GREEDY | ZSTD_LAZY | ZSTD_LAZY2 | ZSTD_BTLAZY2 => { + let row_log = self.cparams.searchLog.clamp(4, 6); + let mut active_next = 0u32; + let mut active_lazy_skipping = 0; + let mut active_hash_salt_entropy = 0u32; + let dict_state = DictLazyStateProjection { + hashTable: self.dict_hash_table.as_mut_ptr(), + chainTable: self.dict_chain_table.as_mut_ptr(), + tagTable: self.dict_tag_table.as_mut_ptr(), + hashCache: ptr::null_mut(), + base: self.dict_storage.as_ptr(), + dictBase: self.dict_storage.as_ptr(), + nextSrc: self.dict_storage.as_ptr().add(self.dict_storage.len()), + dictLimit: DICT_WINDOW_START_INDEX, + lowLimit: DICT_WINDOW_START_INDEX, + loadedDictEnd: self.dict_storage.len() as u32, + nextToUpdate: ptr::addr_of_mut!(self.dict_next_to_update), + lazySkipping: ptr::null_mut(), + hashLog: self.cparams.hashLog, + chainLog: self.cparams.chainLog, + minMatch: self.cparams.minMatch, + searchLog: self.cparams.searchLog, + windowLog: self.cparams.windowLog, + rowHashLog: self.cparams.hashLog - row_log, + hashSalt: 0, + hashSaltEntropy: ptr::null_mut(), + dictMatchState: ptr::null(), + }; + let dict_state = if self.has_dictionary { + ptr::addr_of!(dict_state) + } else { + ptr::null() + }; + let mut active_state = DictLazyStateProjection { + hashTable: self.active_hash_table.as_mut_ptr(), + chainTable: self.active_chain_table.as_mut_ptr(), + tagTable: self.active_tag_table.as_mut_ptr(), + hashCache: self.active_hash_cache.as_mut_ptr(), + base: source, + dictBase: source, + nextSrc: source.add(src_size), + dictLimit: 0, + lowLimit: 0, + loadedDictEnd: 0, + nextToUpdate: ptr::addr_of_mut!(active_next), + lazySkipping: ptr::addr_of_mut!(active_lazy_skipping), + hashLog: self.cparams.hashLog, + chainLog: self.cparams.chainLog, + minMatch: self.cparams.minMatch, + searchLog: self.cparams.searchLog, + windowLog: self.cparams.windowLog, + rowHashLog: self.cparams.hashLog - row_log, + hashSalt: 0, + hashSaltEntropy: ptr::addr_of_mut!(active_hash_salt_entropy), + dictMatchState: dict_state, + }; + let search_method = if self.row_match_finder { + ZSTD_SEARCH_ROW_HASH + } else if self.cparams.strategy == ZSTD_BTLAZY2 { + ZSTD_SEARCH_BINARY_TREE + } else { + ZSTD_SEARCH_HASH_CHAIN + }; + let depth = match self.cparams.strategy { + ZSTD_GREEDY => 0, + ZSTD_LAZY => 1, + ZSTD_LAZY2 | ZSTD_BTLAZY2 => 2, + _ => 0, + }; + crate::zstd_lazy::ZSTD_rust_compressBlock_lazy( + (&mut active_state as *mut DictLazyStateProjection).cast(), + (&mut self.seq_store as *mut SeqStore_t).cast(), + reps.as_mut_ptr(), + src, + src_size, + search_method, + depth, + dict_mode, + ) + } + ZSTD_BTOPT | ZSTD_BTULTRA | ZSTD_BTULTRA2 => { + let mut window_base = source; + let mut window_dict_limit = 0u32; + let mut window_low_limit = 0u32; + let mut active_next = 0u32; + let dict_state = Self::make_opt_projection( + self.cparams, + self.dict_hash_table.as_mut_ptr(), + self.dict_chain_table.as_mut_ptr(), + ptr::null_mut(), + ptr::addr_of_mut!(self.dict_next_to_update), + self.dict_storage.as_ptr(), + self.dict_storage.as_ptr(), + self.dict_storage.as_ptr().add(self.dict_storage.len()), + DICT_WINDOW_START_INDEX, + DICT_WINDOW_START_INDEX, + self.dict_storage.len() as u32, + ptr::null_mut(), + ptr::null(), + ); + let dict_state = if self.has_dictionary { + ptr::addr_of!(dict_state) + } else { + ptr::null() + }; + let mut active_state = Self::make_opt_projection( + self.cparams, + self.active_hash_table.as_mut_ptr(), + self.active_chain_table.as_mut_ptr(), + self.active_hash_table3.as_mut_ptr(), + ptr::addr_of_mut!(active_next), + source, + source, + source.add(src_size), + 0, + 0, + 0, + ptr::addr_of_mut!(self.active_opt_state), + dict_state, + ); + active_state.window_base = ptr::addr_of_mut!(window_base); + active_state.window_dict_limit = ptr::addr_of_mut!(window_dict_limit); + active_state.window_low_limit = ptr::addr_of_mut!(window_low_limit); + crate::zstd_opt::ZSTD_rust_compressBlock_opt( + (&mut active_state as *mut DictOptStateProjectionView).cast(), + (&mut self.seq_store as *mut SeqStore_t).cast(), + reps.as_mut_ptr(), + src, + src_size, + if self.cparams.strategy == ZSTD_BTOPT { + 0 + } else { + 2 + }, + dict_mode, + ) + } + _ => return false, + } + }; + if last_literals > src_size { + return false; + } + if last_literals != 0 { + unsafe { + ptr::copy_nonoverlapping( + source.add(src_size - last_literals), + self.seq_store.lit, + last_literals, + ); + self.seq_store.lit = self.seq_store.lit.add(last_literals); + } + } + + let mut previous_block = + unsafe { MaybeUninit::::zeroed().assume_init() }; + let mut next_block = + unsafe { MaybeUninit::::zeroed().assume_init() }; + unsafe { + ZSTD_rust_resetCompressedBlockState(&mut previous_block); + ZSTD_rust_resetCompressedBlockState(&mut next_block); + } + let mut superblock_workspace = [0u64; TMP_WORKSPACE_SIZE / size_of::()]; + let compressed = unsafe { + ZSTD_rust_compressSuperBlock( + (&self.seq_store as *const SeqStore_t).cast(), + (&previous_block as *const ZSTD_compressedBlockState_t).cast(), + (&mut next_block as *mut ZSTD_compressedBlockState_t).cast(), + self.cparams.strategy, + 0, + superblock_workspace.as_mut_ptr().cast(), + TMP_WORKSPACE_SIZE, + 0, + self.cparams.windowLog, + 0, + workplace.as_mut_ptr().cast(), + workplace.len(), + src, + src_size, + 0, + ) + }; + !ERR_isError(compressed) && compressed != 0 + } +} + fn count_entropy_stats( - cdict: *mut ZSTD_CDict, - cctx: *mut ZSTD_CCtx, + context: &mut DictEntropyContext, workplace: &mut [u8], params: &ZSTD_parameters, count_lit: &mut [u32; 256], @@ -549,28 +1308,12 @@ fn count_entropy_stats( lit_counts: &mut [u32; MAX_LL + 1], rep_offsets: &mut [u32; MAXREPOFFSET], src: *const c_void, - mut src_size: usize, + src_size: usize, ) { - let block_size_max = ZSTD_BLOCKSIZE_MAX.min(1usize << params.cParams.windowLog); - src_size = src_size.min(block_size_max); - let begin = unsafe { ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict) }; - if ERR_isError(begin) { + if !context.compress_sample(workplace, params, src, src_size) { return; } - let compressed = unsafe { - ZSTD_compressBlock_deprecated( - cctx, - workplace.as_mut_ptr().cast(), - workplace.len(), - src, - src_size, - ) - }; - if ERR_isError(compressed) || compressed == 0 { - return; - } - - let store = unsafe { &*ZSTD_getSeqStore(cctx) }; + let store = &mut context.seq_store; let literal_count = unsafe { store.lit.offset_from(store.litStart) as usize }; for byte in unsafe { std::slice::from_raw_parts(store.litStart, literal_count) } { count_lit[*byte as usize] += 1; @@ -647,32 +1390,53 @@ unsafe fn analyze_entropy( } else { compression_level }; - let params = unsafe { ZSTD_getParams(level, average_sample_size as u64, dict_buffer_size) }; - let cdict = unsafe { - ZSTD_createCDict_advanced( - dict_buffer.cast(), - dict_buffer_size, - ZSTD_DLM_BY_REF, - ZSTD_DCT_RAW_CONTENT, - params.cParams, - ZSTD_DEFAULT_CMEM, - ) + let src_size_hint = if average_sample_size == 0 { + u64::MAX + } else { + average_sample_size as u64 }; - let cctx = unsafe { ZSTD_createCCtx() }; - let mut workplace = vec![0u8; ZSTD_BLOCKSIZE_MAX]; - if cdict.is_null() || cctx.is_null() { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } + let selected_params = ZSTD_rust_params_selectCParams( + level, + src_size_hint, + dict_buffer_size, + ZSTD_RUST_CPM_UNKNOWN, + ); + let analysis_cparams = ZSTD_rust_params_adjustCParams( + selected_params, + src_size_hint, + dict_buffer_size, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_AUTO, + ); + let params = ZSTD_rust_params_makeParams(analysis_cparams); + let cdict_cparams = ZSTD_rust_params_getCParamsFromCCtxParams( + 0, + 0, + u64::MAX, + dict_buffer_size, + ZSTD_RUST_CPM_CREATE_CDICT, + ZSTD_RUST_PS_AUTO, + 0, + analysis_cparams, + ZSTD_RUST_PS_AUTO, + 0, + ); + let mut context = match DictEntropyContext::new(cdict_cparams, dict_buffer, dict_buffer_size) { + Ok(context) => context, + Err(()) => return dictionary_error(ZstdErrorCode::MemoryAllocation), + }; + let mut workplace = match try_zeroed_vec(ZSTD_BLOCKSIZE_MAX) { + Ok(workplace) => workplace, + Err(()) => return dictionary_error(ZstdErrorCode::MemoryAllocation), + }; + if src_buffer.is_null() && total_src_size != 0 { return dictionary_error(ZstdErrorCode::MemoryAllocation); } let mut source_offset = 0usize; for &sample_size in file_sizes { count_entropy_stats( - cdict, - cctx, + &mut context, &mut workplace, ¶ms, &mut count_lit, @@ -700,10 +1464,6 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(written) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return written; } if written == 8 { @@ -724,10 +1484,6 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(written) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return written; } } @@ -753,10 +1509,6 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(normalized) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return normalized; } off_log = normalized as u32; @@ -774,10 +1526,6 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(normalized) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return normalized; } match_log = normalized as u32; @@ -795,10 +1543,6 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(normalized) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return normalized; } lit_log = normalized as u32; @@ -817,10 +1561,6 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(entropy_size) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return entropy_size; } dst = unsafe { dst.add(entropy_size) }; @@ -835,10 +1575,6 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(header_size) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return header_size; } entropy_size += header_size; @@ -854,10 +1590,6 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(header_size) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return header_size; } entropy_size += header_size; @@ -873,28 +1605,18 @@ unsafe fn analyze_entropy( ) }; if ERR_isError(header_size) { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return header_size; } entropy_size += header_size; dst = unsafe { dst.add(header_size) }; remaining -= header_size; if remaining < 12 { - unsafe { - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); - } return dictionary_error(ZstdErrorCode::DstSizeTooSmall); } unsafe { MEM_writeLE32(dst.cast(), 1); MEM_writeLE32(dst.add(4).cast(), 4); MEM_writeLE32(dst.add(8).cast(), 8); - ZSTD_freeCDict(cdict); - ZSTD_freeCCtx(cctx); } entropy_size + 12 } @@ -1280,4 +2002,43 @@ mod tests { 0 ); } + + #[test] + fn entropy_context_emits_sequences_for_a_repetitive_sample() { + let dictionary = b"the quick brown fox jumps over the lazy dog. "; + let sample: Vec = (0..512) + .map(|index| dictionary[index % dictionary.len()]) + .collect(); + let selected = ZSTD_rust_params_selectCParams( + ZSTD_CLEVEL_DEFAULT, + sample.len() as u64, + dictionary.len(), + ZSTD_RUST_CPM_UNKNOWN, + ); + let cparams = ZSTD_rust_params_adjustCParams( + selected, + sample.len() as u64, + dictionary.len(), + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_AUTO, + ); + let params = ZSTD_rust_params_makeParams(cparams); + let mut context = DictEntropyContext::new(cparams, dictionary.as_ptr(), dictionary.len()) + .expect("the small dictionary context should allocate"); + let mut workplace = vec![0u8; ZSTD_BLOCKSIZE_MAX]; + + assert!(context.compress_sample( + &mut workplace, + ¶ms, + sample.as_ptr().cast(), + sample.len(), + )); + let sequence_count = unsafe { + context + .seq_store + .sequences + .offset_from(context.seq_store.sequencesStart) + }; + assert!(sequence_count > 0); + } }