diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 749fe4105..e60b4f5db 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -655,6 +655,13 @@ size_t ZSTD_rust_noCompressBlock(void* dst, size_t dstCapacity, size_t ZSTD_rust_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock); U32 ZSTD_rust_windowCorrectOverflow(U32 curr, U32 cycleLog, U32 maxDist); +U32 ZSTD_rust_windowCanOverflowCorrect(U32 curr, U32 cycleLog, U32 maxDist, + U32 loadedDictEnd, + U32 nbOverflowCorrections); +U32 ZSTD_rust_windowNeedOverflowCorrection(U32 curr, + U32 canOverflowCorrect, + U32 currentMax, + int overflowCorrectFrequently); /* ZSTD_minGain() : @@ -1084,28 +1091,10 @@ MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, U32 loadedDictEnd, void const* src) { - U32 const cycleSize = 1u << cycleLog; U32 const curr = (U32)((BYTE const*)src - window.base); - U32 const minIndexToOverflowCorrect = cycleSize - + MAX(maxDist, cycleSize) - + ZSTD_WINDOW_START_INDEX; - - /* Adjust the min index to backoff the overflow correction frequency, - * so we don't waste too much CPU in overflow correction. If this - * computation overflows we don't really care, we just need to make - * sure it is at least minIndexToOverflowCorrect. - */ - U32 const adjustment = window.nbOverflowCorrections + 1; - U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, - minIndexToOverflowCorrect); - U32 const indexLargeEnough = curr > adjustedIndex; - - /* Only overflow correct early if the dictionary is invalidated already, - * so we don't hurt compression ratio. - */ - U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; - - return indexLargeEnough && dictionaryInvalidated; + return ZSTD_rust_windowCanOverflowCorrect( + curr, cycleLog, maxDist, loadedDictEnd, + window.nbOverflowCorrections); } /** @@ -1121,12 +1110,12 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, void const* srcEnd) { U32 const curr = (U32)((BYTE const*)srcEnd - window.base); - if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { - if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { - return 1; - } - } - return curr > ZSTD_CURRENT_MAX; + U32 const canOverflowCorrect = + ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, + loadedDictEnd, src); + return ZSTD_rust_windowNeedOverflowCorrection( + curr, canOverflowCorrect, ZSTD_CURRENT_MAX, + ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY); } /** diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 3e1837e81..076839a5a 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -410,6 +410,76 @@ pub extern "C" fn ZSTD_rust_windowCorrectOverflow(curr: u32, cycle_log: u32, max window_correct_overflow(curr, cycle_log, max_dist) } +#[inline] +fn window_can_overflow_correct( + curr: u32, + cycle_log: u32, + max_dist: u32, + loaded_dict_end: u32, + nb_overflow_corrections: u32, +) -> bool { + let cycle_size = 1u32.wrapping_shl(cycle_log); + let min_index_to_overflow_correct = cycle_size + .wrapping_add(max_dist.max(cycle_size)) + .wrapping_add(ZSTD_WINDOW_START_INDEX); + let adjustment = nb_overflow_corrections.wrapping_add(1); + let adjusted_index = min_index_to_overflow_correct + .wrapping_mul(adjustment) + .max(min_index_to_overflow_correct); + let index_large_enough = curr > adjusted_index; + let dictionary_invalidated = curr > max_dist.wrapping_add(loaded_dict_end); + index_large_enough && dictionary_invalidated +} + +/// Return whether the C-owned window state should enter overflow correction. +/// +/// C retains both pointer subtractions and the `ZSTD_window_t` state. The +/// early-correction result is supplied as a scalar because it is based on the +/// block-start index, while `curr` is the block-end index used by the normal +/// current-limit fallback. The current limit and fuzzing policy are supplied +/// by C so their 32/64-bit and build-mode choices remain authoritative. +#[inline] +fn window_need_overflow_correction( + curr: u32, + can_overflow_correct: bool, + current_max: u32, + overflow_correct_frequently: bool, +) -> bool { + (overflow_correct_frequently && can_overflow_correct) || curr > current_max +} + +#[no_mangle] +pub extern "C" fn ZSTD_rust_windowCanOverflowCorrect( + curr: u32, + cycle_log: u32, + max_dist: u32, + loaded_dict_end: u32, + nb_overflow_corrections: u32, +) -> u32 { + window_can_overflow_correct( + curr, + cycle_log, + max_dist, + loaded_dict_end, + nb_overflow_corrections, + ) as u32 +} + +#[no_mangle] +pub extern "C" fn ZSTD_rust_windowNeedOverflowCorrection( + curr: u32, + can_overflow_correct: u32, + current_max: u32, + overflow_correct_frequently: c_int, +) -> u32 { + window_need_overflow_correction( + curr, + can_overflow_correct != 0, + current_max, + overflow_correct_frequently != 0, + ) as u32 +} + #[inline] fn next_input_size_hint( in_buffer_mode: c_int, @@ -1440,6 +1510,62 @@ mod tests { ); } + #[test] + fn window_overflow_need_uses_the_explicit_current_max_boundary() { + assert!(!window_need_overflow_correction(1024, false, 1024, false)); + assert!(window_need_overflow_correction(1025, false, 1024, false)); + assert_eq!(ZSTD_rust_windowNeedOverflowCorrection(1024, 1, 1024, 0), 0); + assert_eq!(ZSTD_rust_windowNeedOverflowCorrection(1025, 0, 1024, 0), 1); + } + + #[test] + fn window_overflow_need_honors_frequent_dictionary_policy() { + let can_correct = window_can_overflow_correct(27, 3, 16, 0, 0); + let dictionary_still_valid = window_can_overflow_correct(27, 3, 16, 20, 0); + assert!(can_correct); + assert!(!dictionary_still_valid); + assert!(window_need_overflow_correction(27, can_correct, 1024, true)); + assert!(!window_need_overflow_correction( + 27, + dictionary_still_valid, + 1024, + true + )); + assert_eq!( + ZSTD_rust_windowNeedOverflowCorrection(27, can_correct as u32, 1024, 1), + 1 + ); + assert_eq!( + ZSTD_rust_windowNeedOverflowCorrection(27, dictionary_still_valid as u32, 1024, 1,), + 0 + ); + assert!(!window_need_overflow_correction(27, true, 1024, false)); + } + + #[test] + fn window_overflow_need_scales_the_frequency_with_correction_count() { + assert!(window_can_overflow_correct(27, 3, 16, 0, 0)); + assert!(!window_can_overflow_correct(27, 3, 16, 0, 1)); + assert!(window_can_overflow_correct(53, 3, 16, 0, 1)); + assert_eq!(ZSTD_rust_windowCanOverflowCorrect(53, 3, 16, 0, 1), 1); + } + + #[test] + fn window_overflow_need_preserves_u32_wrap_boundaries() { + assert!(window_can_overflow_correct( + 3, + 31, + 0x8000_0000, + 0x8000_0000, + 0, + )); + assert!(window_can_overflow_correct(27, 3, 16, 0, u32::MAX,)); + assert_eq!( + ZSTD_rust_windowCanOverflowCorrect(3, 31, 0x8000_0000, 0x8000_0000, 0), + 1 + ); + } + #[test] fn dict_too_big_uses_a_strict_chunk_size_boundary() { assert!(!dict_too_big(0));