diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 90a3fa1d5..4c2931374 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -6412,8 +6412,17 @@ static U32 ZSTD_rust_overflowCorrect_correct(void* context, const void* src) state->params->cParams.chainLog, state->params->cParams.strategy); U32 const maxDist = (U32)1 << state->params->cParams.windowLog; - return ZSTD_window_correctOverflow( - &state->matchState->window, cycleLog, maxDist, src); + ZSTD_rust_windowOverflowState windowState = { + &state->matchState->window.base, + &state->matchState->window.dictBase, + &state->matchState->window.dictLimit, + &state->matchState->window.lowLimit, + &state->matchState->window.nbOverflowCorrections, + }; + U32 const curr = (U32)((BYTE const*)src - state->matchState->window.base); + return ZSTD_rust_windowCorrectOverflowState( + &windowState, curr, cycleLog, maxDist, + ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY); } static void ZSTD_rust_overflowCorrect_markTablesDirty(void* context) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index f4e1809ad..1529d394d 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -669,6 +669,21 @@ typedef char ZSTD_rust_window_update_state_layout[ && offsetof(ZSTD_rust_windowUpdateState, dictLimit) == 3 * sizeof(void*) && offsetof(ZSTD_rust_windowUpdateState, lowLimit) == 3 * sizeof(void*) + sizeof(U32) && sizeof(ZSTD_rust_windowUpdateState) == 3 * sizeof(void*) + 2 * sizeof(U32)) ? 1 : -1]; +typedef struct { + const BYTE** base; + const BYTE** dictBase; + U32* dictLimit; + U32* lowLimit; + U32* nbOverflowCorrections; +} ZSTD_rust_windowOverflowState; +typedef char ZSTD_rust_window_overflow_state_layout[ + (offsetof(ZSTD_rust_windowOverflowState, base) == 0 + && offsetof(ZSTD_rust_windowOverflowState, dictBase) == sizeof(void*) + && offsetof(ZSTD_rust_windowOverflowState, dictLimit) == 2 * sizeof(void*) + && offsetof(ZSTD_rust_windowOverflowState, lowLimit) == 3 * sizeof(void*) + && offsetof(ZSTD_rust_windowOverflowState, nbOverflowCorrections) + == 4 * sizeof(void*) + && sizeof(ZSTD_rust_windowOverflowState) == 5 * sizeof(void*)) ? 1 : -1]; typedef struct { U32 lowLimit; U32 dictLimit; @@ -694,6 +709,9 @@ U32 ZSTD_rust_windowUpdate(ZSTD_rust_windowUpdateState* state, int forceNonContiguous); void ZSTD_rust_windowClear(size_t endT, U32* lowLimit, U32* dictLimit); U32 ZSTD_rust_windowCorrectOverflow(U32 curr, U32 cycleLog, U32 maxDist); +U32 ZSTD_rust_windowCorrectOverflowState( + ZSTD_rust_windowOverflowState* state, U32 curr, + U32 cycleLog, U32 maxDist, int overflowCorrectFrequently); U32 ZSTD_rust_windowCanOverflowCorrect(U32 curr, U32 cycleLog, U32 maxDist, U32 loadedDictEnd, U32 nbOverflowCorrections); @@ -1143,84 +1161,6 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY); } -/** - * ZSTD_window_correctOverflow(): - * Reduces the indices to protect from index overflow. - * Returns the correction made to the indices, which must be applied to every - * stored index. - * - * The least significant cycleLog bits of the indices must remain the same, - * which may be 0. Every index up to maxDist in the past must be valid. - */ -MEM_STATIC -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, - U32 maxDist, void const* src) -{ - /* preemptive overflow correction: - * 1. correction is large enough: - * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) - * > 1<<29 - * - * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: - * After correction, current is less than (1<base < 1<<32. - * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); - U32 const correction = ZSTD_rust_windowCorrectOverflow(curr, cycleLog, maxDist); - U32 const newCurrent = curr - correction; - /* maxDist must be a power of two so that: - * (newCurrent & cycleMask) == (curr & cycleMask), where cycleMask is - * (1 << cycleLog) - 1 - * This is required to not corrupt the chains / binary tree. - */ - assert((maxDist & (maxDist - 1)) == 0); - assert((curr & ((1u << cycleLog) - 1)) == - (newCurrent & ((1u << cycleLog) - 1))); - assert(curr > newCurrent); - if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { - /* Loose bound, should be around 1<<29 (see above) */ - assert(correction > 1<<28); - } - - window->base += correction; - window->dictBase += correction; - if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) { - window->lowLimit = ZSTD_WINDOW_START_INDEX; - } else { - window->lowLimit -= correction; - } - if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) { - window->dictLimit = ZSTD_WINDOW_START_INDEX; - } else { - window->dictLimit -= correction; - } - - /* Ensure we can still reference the full window. */ - assert(newCurrent >= maxDist); - assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX); - /* Ensure that lowLimit and dictLimit didn't underflow. */ - assert(window->lowLimit <= newCurrent); - assert(window->dictLimit <= newCurrent); - (void)newCurrent; - - ++window->nbOverflowCorrections; - - DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, - window->lowLimit); - return correction; -} - /** * ZSTD_window_enforceMaxDist(): * Updates lowLimit so that: diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 4459c3a5e..44164a51e 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -7921,6 +7921,100 @@ pub extern "C" fn ZSTD_rust_windowCorrectOverflow(curr: u32, cycle_log: u32, max window_correct_overflow(curr, cycle_log, max_dist) } +/// Project the private window fields needed by overflow correction. +/// +/// C retains the containing `ZSTD_window_t` and computes the source index +/// against its private base. Rust owns the stateful rebase, including pointer +/// advancement, limit clamping, and the correction counter. +#[repr(C)] +pub struct ZSTD_rust_windowOverflowState { + pub base: *mut *const c_void, + pub dict_base: *mut *const c_void, + pub dict_limit: *mut c_uint, + pub low_limit: *mut c_uint, + pub nb_overflow_corrections: *mut c_uint, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_windowOverflowState, base) == 0); + assert!(offset_of!(ZSTD_rust_windowOverflowState, dict_base) == size_of::()); + assert!(offset_of!(ZSTD_rust_windowOverflowState, dict_limit) == 2 * size_of::()); + assert!(offset_of!(ZSTD_rust_windowOverflowState, low_limit) == 3 * size_of::()); + assert!( + offset_of!(ZSTD_rust_windowOverflowState, nb_overflow_corrections) + == 4 * size_of::() + ); + assert!(size_of::() == 5 * size_of::()); +}; + +/// Apply the stateful window rebase after C has decided correction is needed. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_windowCorrectOverflowState( + state: *mut ZSTD_rust_windowOverflowState, + curr: u32, + cycle_log: u32, + max_dist: u32, + overflow_correct_frequently: c_int, +) -> c_uint { + if state.is_null() { + return 0; + } + let state = unsafe { &mut *state }; + if state.base.is_null() + || state.dict_base.is_null() + || state.dict_limit.is_null() + || state.low_limit.is_null() + || state.nb_overflow_corrections.is_null() + { + return 0; + } + + let correction = window_correct_overflow(curr, cycle_log, max_dist); + let new_current = curr.wrapping_sub(correction); + let cycle_mask = 1u32.wrapping_shl(cycle_log).wrapping_sub(1); + debug_assert!(max_dist & max_dist.wrapping_sub(1) == 0); + debug_assert_eq!(curr & cycle_mask, new_current & cycle_mask); + debug_assert!(curr > new_current); + if overflow_correct_frequently == 0 { + debug_assert!(correction > 1 << 28); + } + + unsafe { + let base = &mut *state.base; + *base = (*base) + .cast::() + .wrapping_add(correction as usize) + .cast(); + let dict_base = &mut *state.dict_base; + *dict_base = (*dict_base) + .cast::() + .wrapping_add(correction as usize) + .cast(); + + let limit_adjustment = correction.wrapping_add(ZSTD_WINDOW_START_INDEX); + let low_limit = &mut *state.low_limit; + if *low_limit < limit_adjustment { + *low_limit = ZSTD_WINDOW_START_INDEX; + } else { + *low_limit = (*low_limit).wrapping_sub(correction); + } + let dict_limit = &mut *state.dict_limit; + if *dict_limit < limit_adjustment { + *dict_limit = ZSTD_WINDOW_START_INDEX; + } else { + *dict_limit = (*dict_limit).wrapping_sub(correction); + } + + debug_assert!(new_current >= max_dist); + debug_assert!(new_current.wrapping_sub(max_dist) >= ZSTD_WINDOW_START_INDEX); + debug_assert!(*low_limit <= new_current); + debug_assert!(*dict_limit <= new_current); + *state.nb_overflow_corrections = (*state.nb_overflow_corrections).wrapping_add(1); + } + + correction +} + #[inline] fn window_can_overflow_correct( curr: u32, @@ -16625,6 +16719,36 @@ mod tests { ); } + #[test] + fn window_correction_rebases_projected_state() { + let buffer = [0u8; 512]; + let initial_base = buffer.as_ptr().cast::(); + let initial_dict_base = unsafe { buffer.as_ptr().add(16).cast::() }; + let mut base = initial_base; + let mut dict_base = initial_dict_base; + let mut dict_limit = 0x110; + let mut low_limit = 0x120; + let mut nb_overflow_corrections = 2; + let mut state = ZSTD_rust_windowOverflowState { + base: &mut base, + dict_base: &mut dict_base, + dict_limit: &mut dict_limit, + low_limit: &mut low_limit, + nb_overflow_corrections: &mut nb_overflow_corrections, + }; + + let correction = unsafe { + ZSTD_rust_windowCorrectOverflowState(&mut state, 0x100, 3, 8, 1) + }; + + assert_eq!(correction, 0xf0); + assert_eq!(base as usize, initial_base as usize + 0xf0); + assert_eq!(dict_base as usize, initial_dict_base as usize + 0xf0); + assert_eq!(dict_limit, 0x20); + assert_eq!(low_limit, 0x30); + assert_eq!(nb_overflow_corrections, 3); + } + #[test] fn window_overflow_need_uses_the_explicit_current_max_boundary() { assert!(!window_need_overflow_correction(1024, false, 1024, false));