diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e694a25fd..95a92d8b7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -67,8 +67,10 @@ size_t ZSTD_compressStream2_c(ZSTD_CCtx* cctx, ZSTD_EndDirective endOp); int ZSTD_rust_simpleCompress2Level(const void* cctx); int ZSTD_rust_simpleCompressStream2Level(const void* cctx); -void ZSTD_rust_reduceTable(U32* table, U32 size, U32 reducerValue, - int preserveMark); +void ZSTD_rust_reduceIndex(U32* hashTable, U32 hashSize, + U32* chainTable, U32 chainSize, + U32* hashTable3, U32 hashSize3, + U32 reducerValue, int preserveChainMark); void ZSTD_rust_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, int tagged); U64 ZSTD_rust_advanceHashSalt(U64 hashSalt, U64 hashSaltEntropy); @@ -2116,42 +2118,31 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long } -/*! ZSTD_reduceTable() : - * reduce table indexes by `reducerValue`, or squash to zero. - * PreserveMark preserves "unsorted mark" for btlazy2 strategy. - * It must be set to a clear 0/1 value, to remove branch during inlining. - * Presume table size is a multiple of 16 cells - * to help auto-vectorization */ -static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue) -{ - ZSTD_rust_reduceTable(table, size, reducerValue, 0); -} - -static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue) -{ - ZSTD_rust_reduceTable(table, size, reducerValue, 1); -} - /*! ZSTD_reduceIndex() : * rescale all indexes to avoid future overflow (indexes are U32) */ static void ZSTD_reduceIndex (ZSTD_MatchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue) { - { U32 const hSize = (U32)1 << params->cParams.hashLog; - ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); - } + U32 const hSize = (U32)1 << params->cParams.hashLog; + U32* chainTable = NULL; + U32 chainSize = 0; + U32* hashTable3 = NULL; + U32 hashSize3 = 0; if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { - U32 const chainSize = (U32)1 << params->cParams.chainLog; - if (params->cParams.strategy == ZSTD_btlazy2) - ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); - else - ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + chainTable = ms->chainTable; + chainSize = (U32)1 << params->cParams.chainLog; } if (ms->hashLog3) { - U32 const h3Size = (U32)1 << ms->hashLog3; - ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); + hashTable3 = ms->hashTable3; + hashSize3 = (U32)1 << ms->hashLog3; } + + ZSTD_rust_reduceIndex(ms->hashTable, hSize, + chainTable, chainSize, + hashTable3, hashSize3, + reducerValue, + params->cParams.strategy == ZSTD_btlazy2); } diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 812f66ecf..3e1837e81 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -708,8 +708,8 @@ fn reduce_table_internal(table: &mut [u32], reducer_value: u32, preserve_mark: b /// Rust implementation of the C match-table reduction leaf. /// -/// The C wrappers select the ordinary or btlazy2 policy by passing a clear -/// zero/one `preserve_mark` value; `ZSTD_reduceIndex` remains C-owned. +/// The C caller supplies a clear zero/one `preserve_mark` value for the +/// strategy-specific btlazy2 policy. #[no_mangle] pub unsafe extern "C" fn ZSTD_rust_reduceTable( table: *mut u32, @@ -728,6 +728,35 @@ pub unsafe extern "C" fn ZSTD_rust_reduceTable( reduce_table_internal(table, reducer_value, preserve_mark != 0); } +/// Reduce the match tables selected by C's stateful overflow-correction path. +/// +/// C retains match-state access and table selection. Each non-zero size is +/// paired with a validated mutable table pointer; zero-sized optional tables +/// may use null pointers. Only the chain table receives the btlazy2 marker +/// policy. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_reduceIndex( + hash_table: *mut u32, + hash_size: u32, + chain_table: *mut u32, + chain_size: u32, + hash_table3: *mut u32, + hash_size3: u32, + reducer_value: u32, + preserve_chain_mark: c_int, +) { + debug_assert!(!hash_table.is_null() || hash_size == 0); + debug_assert!(!chain_table.is_null() || chain_size == 0); + debug_assert!(!hash_table3.is_null() || hash_size3 == 0); + debug_assert!(preserve_chain_mark == 0 || preserve_chain_mark == 1); + + unsafe { + ZSTD_rust_reduceTable(hash_table, hash_size, reducer_value, 0); + ZSTD_rust_reduceTable(chain_table, chain_size, reducer_value, preserve_chain_mark); + ZSTD_rust_reduceTable(hash_table3, hash_size3, reducer_value, 0); + } +} + /// Copies a CDict match table into a CCtx, removing short-cache tags when the /// C-owned compression parameters say the source table is tagged. #[no_mangle] @@ -1266,6 +1295,64 @@ mod tests { assert_eq!(table[ZSTD_ROWSIZE * 2 - 1], u32::MAX - 4); } + #[test] + fn reduce_index_accepts_zero_sized_optional_tables() { + let mut hash_table = [0u32; ZSTD_ROWSIZE]; + hash_table[..4].copy_from_slice(&[1, 2, 5, u32::MAX]); + + unsafe { + ZSTD_rust_reduceIndex( + hash_table.as_mut_ptr(), + hash_table.len() as u32, + ptr::null_mut(), + 0, + ptr::null_mut(), + 0, + 3, + 0, + ); + ZSTD_rust_reduceIndex( + ptr::null_mut(), + 0, + ptr::null_mut(), + 0, + ptr::null_mut(), + 0, + u32::MAX, + 1, + ); + } + + assert_eq!(&hash_table[..4], &[0, 0, 2, u32::MAX - 3]); + } + + #[test] + fn reduce_index_preserves_the_marker_only_in_the_chain_table() { + let mut hash_table = [0u32; ZSTD_ROWSIZE]; + let mut chain_table = [0u32; ZSTD_ROWSIZE]; + let mut hash_table3 = [0u32; ZSTD_ROWSIZE]; + hash_table[0] = ZSTD_DUBT_UNSORTED_MARK; + chain_table[0] = ZSTD_DUBT_UNSORTED_MARK; + hash_table3[0] = ZSTD_DUBT_UNSORTED_MARK; + + unsafe { + ZSTD_rust_reduceIndex( + hash_table.as_mut_ptr(), + hash_table.len() as u32, + chain_table.as_mut_ptr(), + chain_table.len() as u32, + hash_table3.as_mut_ptr(), + hash_table3.len() as u32, + 3, + 1, + ); + } + + assert_eq!(hash_table[0], 0); + assert_eq!(chain_table[0], ZSTD_DUBT_UNSORTED_MARK); + assert_eq!(hash_table3[0], 0); + } + #[test] fn copy_cdict_table_removes_short_cache_tags() { let source = [0x1234_56ff, 0xdead_beef, 0x0000_0100, u32::MAX];