feat(compress): move window overflow rebase to Rust

The overflow-correction policy and scalar correction arithmetic were already
Rust-owned, but the stateful window rebase still lived in the C
ZSTD_window_correctOverflow() inline. That left pointer advancement, limit
clamping, and the correction counter as an untracked production C algorithm
on every compression path that protects matchfinder indices.

Replace that inline with a five-pointer projection. C continues to compute the
private source index and choose the correction context, while Rust now owns
the rebasing transition: it advances the regular and dictionary bases, clamps
both limits with the original wrapping U32 behavior, checks the correction
invariants in debug builds, and increments the overflow-correction counter.
The C/Rust projection has explicit layout assertions, and the Rust fixture
covers the pointer, limit, and counter updates together.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 -C tests invalidDictionaries
- ulimit -v 41943040; make -j1 -C tests test
- git diff --check
This commit is contained in:
2026-07-21 16:55:33 +02:00
parent 02d3579da3
commit 8aa7156943
3 changed files with 153 additions and 80 deletions
+11 -2
View File
@@ -6412,8 +6412,17 @@ static U32 ZSTD_rust_overflowCorrect_correct(void* context, const void* src)
state->params->cParams.chainLog, state->params->cParams.chainLog,
state->params->cParams.strategy); state->params->cParams.strategy);
U32 const maxDist = (U32)1 << state->params->cParams.windowLog; U32 const maxDist = (U32)1 << state->params->cParams.windowLog;
return ZSTD_window_correctOverflow( ZSTD_rust_windowOverflowState windowState = {
&state->matchState->window, cycleLog, maxDist, src); &state->matchState->window.base,
&state->matchState->window.dictBase,
&state->matchState->window.dictLimit,
&state->matchState->window.lowLimit,
&state->matchState->window.nbOverflowCorrections,
};
U32 const curr = (U32)((BYTE const*)src - state->matchState->window.base);
return ZSTD_rust_windowCorrectOverflowState(
&windowState, curr, cycleLog, maxDist,
ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY);
} }
static void ZSTD_rust_overflowCorrect_markTablesDirty(void* context) static void ZSTD_rust_overflowCorrect_markTablesDirty(void* context)
+18 -78
View File
@@ -669,6 +669,21 @@ typedef char ZSTD_rust_window_update_state_layout[
&& offsetof(ZSTD_rust_windowUpdateState, dictLimit) == 3 * sizeof(void*) && offsetof(ZSTD_rust_windowUpdateState, dictLimit) == 3 * sizeof(void*)
&& offsetof(ZSTD_rust_windowUpdateState, lowLimit) == 3 * sizeof(void*) + sizeof(U32) && offsetof(ZSTD_rust_windowUpdateState, lowLimit) == 3 * sizeof(void*) + sizeof(U32)
&& sizeof(ZSTD_rust_windowUpdateState) == 3 * sizeof(void*) + 2 * sizeof(U32)) ? 1 : -1]; && sizeof(ZSTD_rust_windowUpdateState) == 3 * sizeof(void*) + 2 * sizeof(U32)) ? 1 : -1];
typedef struct {
const BYTE** base;
const BYTE** dictBase;
U32* dictLimit;
U32* lowLimit;
U32* nbOverflowCorrections;
} ZSTD_rust_windowOverflowState;
typedef char ZSTD_rust_window_overflow_state_layout[
(offsetof(ZSTD_rust_windowOverflowState, base) == 0
&& offsetof(ZSTD_rust_windowOverflowState, dictBase) == sizeof(void*)
&& offsetof(ZSTD_rust_windowOverflowState, dictLimit) == 2 * sizeof(void*)
&& offsetof(ZSTD_rust_windowOverflowState, lowLimit) == 3 * sizeof(void*)
&& offsetof(ZSTD_rust_windowOverflowState, nbOverflowCorrections)
== 4 * sizeof(void*)
&& sizeof(ZSTD_rust_windowOverflowState) == 5 * sizeof(void*)) ? 1 : -1];
typedef struct { typedef struct {
U32 lowLimit; U32 lowLimit;
U32 dictLimit; U32 dictLimit;
@@ -694,6 +709,9 @@ U32 ZSTD_rust_windowUpdate(ZSTD_rust_windowUpdateState* state,
int forceNonContiguous); int forceNonContiguous);
void ZSTD_rust_windowClear(size_t endT, U32* lowLimit, U32* dictLimit); void ZSTD_rust_windowClear(size_t endT, U32* lowLimit, U32* dictLimit);
U32 ZSTD_rust_windowCorrectOverflow(U32 curr, U32 cycleLog, U32 maxDist); U32 ZSTD_rust_windowCorrectOverflow(U32 curr, U32 cycleLog, U32 maxDist);
U32 ZSTD_rust_windowCorrectOverflowState(
ZSTD_rust_windowOverflowState* state, U32 curr,
U32 cycleLog, U32 maxDist, int overflowCorrectFrequently);
U32 ZSTD_rust_windowCanOverflowCorrect(U32 curr, U32 cycleLog, U32 maxDist, U32 ZSTD_rust_windowCanOverflowCorrect(U32 curr, U32 cycleLog, U32 maxDist,
U32 loadedDictEnd, U32 loadedDictEnd,
U32 nbOverflowCorrections); U32 nbOverflowCorrections);
@@ -1143,84 +1161,6 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY); ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY);
} }
/**
* ZSTD_window_correctOverflow():
* Reduces the indices to protect from index overflow.
* Returns the correction made to the indices, which must be applied to every
* stored index.
*
* The least significant cycleLog bits of the indices must remain the same,
* which may be 0. Every index up to maxDist in the past must be valid.
*/
MEM_STATIC
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
U32 maxDist, void const* src)
{
/* preemptive overflow correction:
* 1. correction is large enough:
* lowLimit > (3<<29) ==> current > 3<<29 + 1<<windowLog
* 1<<windowLog <= newCurrent < 1<<chainLog + 1<<windowLog
*
* current - newCurrent
* > (3<<29 + 1<<windowLog) - (1<<windowLog + 1<<chainLog)
* > (3<<29) - (1<<chainLog)
* > (3<<29) - (1<<30) (NOTE: chainLog <= 30)
* > 1<<29
*
* 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow:
* After correction, current is less than (1<<chainLog + 1<<windowLog).
* In 64-bit mode we are safe, because we have 64-bit ptrdiff_t.
* In 32-bit mode we are safe, because (chainLog <= 29), so
* ip+ZSTD_CHUNKSIZE_MAX - cctx->base < 1<<32.
* 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
*/
U32 const curr = (U32)((BYTE const*)src - window->base);
U32 const correction = ZSTD_rust_windowCorrectOverflow(curr, cycleLog, maxDist);
U32 const newCurrent = curr - correction;
/* maxDist must be a power of two so that:
* (newCurrent & cycleMask) == (curr & cycleMask), where cycleMask is
* (1 << cycleLog) - 1
* This is required to not corrupt the chains / binary tree.
*/
assert((maxDist & (maxDist - 1)) == 0);
assert((curr & ((1u << cycleLog) - 1)) ==
(newCurrent & ((1u << cycleLog) - 1)));
assert(curr > newCurrent);
if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
/* Loose bound, should be around 1<<29 (see above) */
assert(correction > 1<<28);
}
window->base += correction;
window->dictBase += correction;
if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) {
window->lowLimit = ZSTD_WINDOW_START_INDEX;
} else {
window->lowLimit -= correction;
}
if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) {
window->dictLimit = ZSTD_WINDOW_START_INDEX;
} else {
window->dictLimit -= correction;
}
/* Ensure we can still reference the full window. */
assert(newCurrent >= maxDist);
assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX);
/* Ensure that lowLimit and dictLimit didn't underflow. */
assert(window->lowLimit <= newCurrent);
assert(window->dictLimit <= newCurrent);
(void)newCurrent;
++window->nbOverflowCorrections;
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
window->lowLimit);
return correction;
}
/** /**
* ZSTD_window_enforceMaxDist(): * ZSTD_window_enforceMaxDist():
* Updates lowLimit so that: * Updates lowLimit so that:
+124
View File
@@ -7921,6 +7921,100 @@ pub extern "C" fn ZSTD_rust_windowCorrectOverflow(curr: u32, cycle_log: u32, max
window_correct_overflow(curr, cycle_log, max_dist) window_correct_overflow(curr, cycle_log, max_dist)
} }
/// Project the private window fields needed by overflow correction.
///
/// C retains the containing `ZSTD_window_t` and computes the source index
/// against its private base. Rust owns the stateful rebase, including pointer
/// advancement, limit clamping, and the correction counter.
#[repr(C)]
pub struct ZSTD_rust_windowOverflowState {
pub base: *mut *const c_void,
pub dict_base: *mut *const c_void,
pub dict_limit: *mut c_uint,
pub low_limit: *mut c_uint,
pub nb_overflow_corrections: *mut c_uint,
}
const _: () = {
assert!(offset_of!(ZSTD_rust_windowOverflowState, base) == 0);
assert!(offset_of!(ZSTD_rust_windowOverflowState, dict_base) == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_windowOverflowState, dict_limit) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_windowOverflowState, low_limit) == 3 * size_of::<usize>());
assert!(
offset_of!(ZSTD_rust_windowOverflowState, nb_overflow_corrections)
== 4 * size_of::<usize>()
);
assert!(size_of::<ZSTD_rust_windowOverflowState>() == 5 * size_of::<usize>());
};
/// Apply the stateful window rebase after C has decided correction is needed.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_windowCorrectOverflowState(
state: *mut ZSTD_rust_windowOverflowState,
curr: u32,
cycle_log: u32,
max_dist: u32,
overflow_correct_frequently: c_int,
) -> c_uint {
if state.is_null() {
return 0;
}
let state = unsafe { &mut *state };
if state.base.is_null()
|| state.dict_base.is_null()
|| state.dict_limit.is_null()
|| state.low_limit.is_null()
|| state.nb_overflow_corrections.is_null()
{
return 0;
}
let correction = window_correct_overflow(curr, cycle_log, max_dist);
let new_current = curr.wrapping_sub(correction);
let cycle_mask = 1u32.wrapping_shl(cycle_log).wrapping_sub(1);
debug_assert!(max_dist & max_dist.wrapping_sub(1) == 0);
debug_assert_eq!(curr & cycle_mask, new_current & cycle_mask);
debug_assert!(curr > new_current);
if overflow_correct_frequently == 0 {
debug_assert!(correction > 1 << 28);
}
unsafe {
let base = &mut *state.base;
*base = (*base)
.cast::<u8>()
.wrapping_add(correction as usize)
.cast();
let dict_base = &mut *state.dict_base;
*dict_base = (*dict_base)
.cast::<u8>()
.wrapping_add(correction as usize)
.cast();
let limit_adjustment = correction.wrapping_add(ZSTD_WINDOW_START_INDEX);
let low_limit = &mut *state.low_limit;
if *low_limit < limit_adjustment {
*low_limit = ZSTD_WINDOW_START_INDEX;
} else {
*low_limit = (*low_limit).wrapping_sub(correction);
}
let dict_limit = &mut *state.dict_limit;
if *dict_limit < limit_adjustment {
*dict_limit = ZSTD_WINDOW_START_INDEX;
} else {
*dict_limit = (*dict_limit).wrapping_sub(correction);
}
debug_assert!(new_current >= max_dist);
debug_assert!(new_current.wrapping_sub(max_dist) >= ZSTD_WINDOW_START_INDEX);
debug_assert!(*low_limit <= new_current);
debug_assert!(*dict_limit <= new_current);
*state.nb_overflow_corrections = (*state.nb_overflow_corrections).wrapping_add(1);
}
correction
}
#[inline] #[inline]
fn window_can_overflow_correct( fn window_can_overflow_correct(
curr: u32, curr: u32,
@@ -16625,6 +16719,36 @@ mod tests {
); );
} }
#[test]
fn window_correction_rebases_projected_state() {
let buffer = [0u8; 512];
let initial_base = buffer.as_ptr().cast::<c_void>();
let initial_dict_base = unsafe { buffer.as_ptr().add(16).cast::<c_void>() };
let mut base = initial_base;
let mut dict_base = initial_dict_base;
let mut dict_limit = 0x110;
let mut low_limit = 0x120;
let mut nb_overflow_corrections = 2;
let mut state = ZSTD_rust_windowOverflowState {
base: &mut base,
dict_base: &mut dict_base,
dict_limit: &mut dict_limit,
low_limit: &mut low_limit,
nb_overflow_corrections: &mut nb_overflow_corrections,
};
let correction = unsafe {
ZSTD_rust_windowCorrectOverflowState(&mut state, 0x100, 3, 8, 1)
};
assert_eq!(correction, 0xf0);
assert_eq!(base as usize, initial_base as usize + 0xf0);
assert_eq!(dict_base as usize, initial_dict_base as usize + 0xf0);
assert_eq!(dict_limit, 0x20);
assert_eq!(low_limit, 0x30);
assert_eq!(nb_overflow_corrections, 3);
}
#[test] #[test]
fn window_overflow_need_uses_the_explicit_current_max_boundary() { fn window_overflow_need_uses_the_explicit_current_max_boundary() {
assert!(!window_need_overflow_correction(1024, false, 1024, false)); assert!(!window_need_overflow_correction(1024, false, 1024, false));