feat(compress): move window overflow rebase to Rust
The overflow-correction policy and scalar correction arithmetic were already Rust-owned, but the stateful window rebase still lived in the C ZSTD_window_correctOverflow() inline. That left pointer advancement, limit clamping, and the correction counter as an untracked production C algorithm on every compression path that protects matchfinder indices. Replace that inline with a five-pointer projection. C continues to compute the private source index and choose the correction context, while Rust now owns the rebasing transition: it advances the regular and dictionary bases, clamps both limits with the original wrapping U32 behavior, checks the correction invariants in debug builds, and increments the overflow-correction counter. The C/Rust projection has explicit layout assertions, and the Rust fixture covers the pointer, limit, and counter updates together. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 - ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 -C tests invalidDictionaries - ulimit -v 41943040; make -j1 -C tests test - git diff --check
This commit is contained in:
@@ -6412,8 +6412,17 @@ static U32 ZSTD_rust_overflowCorrect_correct(void* context, const void* src)
|
|||||||
state->params->cParams.chainLog,
|
state->params->cParams.chainLog,
|
||||||
state->params->cParams.strategy);
|
state->params->cParams.strategy);
|
||||||
U32 const maxDist = (U32)1 << state->params->cParams.windowLog;
|
U32 const maxDist = (U32)1 << state->params->cParams.windowLog;
|
||||||
return ZSTD_window_correctOverflow(
|
ZSTD_rust_windowOverflowState windowState = {
|
||||||
&state->matchState->window, cycleLog, maxDist, src);
|
&state->matchState->window.base,
|
||||||
|
&state->matchState->window.dictBase,
|
||||||
|
&state->matchState->window.dictLimit,
|
||||||
|
&state->matchState->window.lowLimit,
|
||||||
|
&state->matchState->window.nbOverflowCorrections,
|
||||||
|
};
|
||||||
|
U32 const curr = (U32)((BYTE const*)src - state->matchState->window.base);
|
||||||
|
return ZSTD_rust_windowCorrectOverflowState(
|
||||||
|
&windowState, curr, cycleLog, maxDist,
|
||||||
|
ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void ZSTD_rust_overflowCorrect_markTablesDirty(void* context)
|
static void ZSTD_rust_overflowCorrect_markTablesDirty(void* context)
|
||||||
|
|||||||
@@ -669,6 +669,21 @@ typedef char ZSTD_rust_window_update_state_layout[
|
|||||||
&& offsetof(ZSTD_rust_windowUpdateState, dictLimit) == 3 * sizeof(void*)
|
&& offsetof(ZSTD_rust_windowUpdateState, dictLimit) == 3 * sizeof(void*)
|
||||||
&& offsetof(ZSTD_rust_windowUpdateState, lowLimit) == 3 * sizeof(void*) + sizeof(U32)
|
&& offsetof(ZSTD_rust_windowUpdateState, lowLimit) == 3 * sizeof(void*) + sizeof(U32)
|
||||||
&& sizeof(ZSTD_rust_windowUpdateState) == 3 * sizeof(void*) + 2 * sizeof(U32)) ? 1 : -1];
|
&& sizeof(ZSTD_rust_windowUpdateState) == 3 * sizeof(void*) + 2 * sizeof(U32)) ? 1 : -1];
|
||||||
|
typedef struct {
|
||||||
|
const BYTE** base;
|
||||||
|
const BYTE** dictBase;
|
||||||
|
U32* dictLimit;
|
||||||
|
U32* lowLimit;
|
||||||
|
U32* nbOverflowCorrections;
|
||||||
|
} ZSTD_rust_windowOverflowState;
|
||||||
|
typedef char ZSTD_rust_window_overflow_state_layout[
|
||||||
|
(offsetof(ZSTD_rust_windowOverflowState, base) == 0
|
||||||
|
&& offsetof(ZSTD_rust_windowOverflowState, dictBase) == sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_windowOverflowState, dictLimit) == 2 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_windowOverflowState, lowLimit) == 3 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_windowOverflowState, nbOverflowCorrections)
|
||||||
|
== 4 * sizeof(void*)
|
||||||
|
&& sizeof(ZSTD_rust_windowOverflowState) == 5 * sizeof(void*)) ? 1 : -1];
|
||||||
typedef struct {
|
typedef struct {
|
||||||
U32 lowLimit;
|
U32 lowLimit;
|
||||||
U32 dictLimit;
|
U32 dictLimit;
|
||||||
@@ -694,6 +709,9 @@ U32 ZSTD_rust_windowUpdate(ZSTD_rust_windowUpdateState* state,
|
|||||||
int forceNonContiguous);
|
int forceNonContiguous);
|
||||||
void ZSTD_rust_windowClear(size_t endT, U32* lowLimit, U32* dictLimit);
|
void ZSTD_rust_windowClear(size_t endT, U32* lowLimit, U32* dictLimit);
|
||||||
U32 ZSTD_rust_windowCorrectOverflow(U32 curr, U32 cycleLog, U32 maxDist);
|
U32 ZSTD_rust_windowCorrectOverflow(U32 curr, U32 cycleLog, U32 maxDist);
|
||||||
|
U32 ZSTD_rust_windowCorrectOverflowState(
|
||||||
|
ZSTD_rust_windowOverflowState* state, U32 curr,
|
||||||
|
U32 cycleLog, U32 maxDist, int overflowCorrectFrequently);
|
||||||
U32 ZSTD_rust_windowCanOverflowCorrect(U32 curr, U32 cycleLog, U32 maxDist,
|
U32 ZSTD_rust_windowCanOverflowCorrect(U32 curr, U32 cycleLog, U32 maxDist,
|
||||||
U32 loadedDictEnd,
|
U32 loadedDictEnd,
|
||||||
U32 nbOverflowCorrections);
|
U32 nbOverflowCorrections);
|
||||||
@@ -1143,84 +1161,6 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
|
|||||||
ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY);
|
ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* ZSTD_window_correctOverflow():
|
|
||||||
* Reduces the indices to protect from index overflow.
|
|
||||||
* Returns the correction made to the indices, which must be applied to every
|
|
||||||
* stored index.
|
|
||||||
*
|
|
||||||
* The least significant cycleLog bits of the indices must remain the same,
|
|
||||||
* which may be 0. Every index up to maxDist in the past must be valid.
|
|
||||||
*/
|
|
||||||
MEM_STATIC
|
|
||||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
|
||||||
U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
|
|
||||||
U32 maxDist, void const* src)
|
|
||||||
{
|
|
||||||
/* preemptive overflow correction:
|
|
||||||
* 1. correction is large enough:
|
|
||||||
* lowLimit > (3<<29) ==> current > 3<<29 + 1<<windowLog
|
|
||||||
* 1<<windowLog <= newCurrent < 1<<chainLog + 1<<windowLog
|
|
||||||
*
|
|
||||||
* current - newCurrent
|
|
||||||
* > (3<<29 + 1<<windowLog) - (1<<windowLog + 1<<chainLog)
|
|
||||||
* > (3<<29) - (1<<chainLog)
|
|
||||||
* > (3<<29) - (1<<30) (NOTE: chainLog <= 30)
|
|
||||||
* > 1<<29
|
|
||||||
*
|
|
||||||
* 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow:
|
|
||||||
* After correction, current is less than (1<<chainLog + 1<<windowLog).
|
|
||||||
* In 64-bit mode we are safe, because we have 64-bit ptrdiff_t.
|
|
||||||
* In 32-bit mode we are safe, because (chainLog <= 29), so
|
|
||||||
* ip+ZSTD_CHUNKSIZE_MAX - cctx->base < 1<<32.
|
|
||||||
* 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
|
|
||||||
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
|
|
||||||
*/
|
|
||||||
U32 const curr = (U32)((BYTE const*)src - window->base);
|
|
||||||
U32 const correction = ZSTD_rust_windowCorrectOverflow(curr, cycleLog, maxDist);
|
|
||||||
U32 const newCurrent = curr - correction;
|
|
||||||
/* maxDist must be a power of two so that:
|
|
||||||
* (newCurrent & cycleMask) == (curr & cycleMask), where cycleMask is
|
|
||||||
* (1 << cycleLog) - 1
|
|
||||||
* This is required to not corrupt the chains / binary tree.
|
|
||||||
*/
|
|
||||||
assert((maxDist & (maxDist - 1)) == 0);
|
|
||||||
assert((curr & ((1u << cycleLog) - 1)) ==
|
|
||||||
(newCurrent & ((1u << cycleLog) - 1)));
|
|
||||||
assert(curr > newCurrent);
|
|
||||||
if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
|
|
||||||
/* Loose bound, should be around 1<<29 (see above) */
|
|
||||||
assert(correction > 1<<28);
|
|
||||||
}
|
|
||||||
|
|
||||||
window->base += correction;
|
|
||||||
window->dictBase += correction;
|
|
||||||
if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) {
|
|
||||||
window->lowLimit = ZSTD_WINDOW_START_INDEX;
|
|
||||||
} else {
|
|
||||||
window->lowLimit -= correction;
|
|
||||||
}
|
|
||||||
if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) {
|
|
||||||
window->dictLimit = ZSTD_WINDOW_START_INDEX;
|
|
||||||
} else {
|
|
||||||
window->dictLimit -= correction;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Ensure we can still reference the full window. */
|
|
||||||
assert(newCurrent >= maxDist);
|
|
||||||
assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX);
|
|
||||||
/* Ensure that lowLimit and dictLimit didn't underflow. */
|
|
||||||
assert(window->lowLimit <= newCurrent);
|
|
||||||
assert(window->dictLimit <= newCurrent);
|
|
||||||
(void)newCurrent;
|
|
||||||
|
|
||||||
++window->nbOverflowCorrections;
|
|
||||||
|
|
||||||
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
|
|
||||||
window->lowLimit);
|
|
||||||
return correction;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ZSTD_window_enforceMaxDist():
|
* ZSTD_window_enforceMaxDist():
|
||||||
* Updates lowLimit so that:
|
* Updates lowLimit so that:
|
||||||
|
|||||||
@@ -7921,6 +7921,100 @@ pub extern "C" fn ZSTD_rust_windowCorrectOverflow(curr: u32, cycle_log: u32, max
|
|||||||
window_correct_overflow(curr, cycle_log, max_dist)
|
window_correct_overflow(curr, cycle_log, max_dist)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Project the private window fields needed by overflow correction.
|
||||||
|
///
|
||||||
|
/// C retains the containing `ZSTD_window_t` and computes the source index
|
||||||
|
/// against its private base. Rust owns the stateful rebase, including pointer
|
||||||
|
/// advancement, limit clamping, and the correction counter.
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct ZSTD_rust_windowOverflowState {
|
||||||
|
pub base: *mut *const c_void,
|
||||||
|
pub dict_base: *mut *const c_void,
|
||||||
|
pub dict_limit: *mut c_uint,
|
||||||
|
pub low_limit: *mut c_uint,
|
||||||
|
pub nb_overflow_corrections: *mut c_uint,
|
||||||
|
}
|
||||||
|
|
||||||
|
const _: () = {
|
||||||
|
assert!(offset_of!(ZSTD_rust_windowOverflowState, base) == 0);
|
||||||
|
assert!(offset_of!(ZSTD_rust_windowOverflowState, dict_base) == size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_windowOverflowState, dict_limit) == 2 * size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_windowOverflowState, low_limit) == 3 * size_of::<usize>());
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_windowOverflowState, nb_overflow_corrections)
|
||||||
|
== 4 * size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(size_of::<ZSTD_rust_windowOverflowState>() == 5 * size_of::<usize>());
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Apply the stateful window rebase after C has decided correction is needed.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_windowCorrectOverflowState(
|
||||||
|
state: *mut ZSTD_rust_windowOverflowState,
|
||||||
|
curr: u32,
|
||||||
|
cycle_log: u32,
|
||||||
|
max_dist: u32,
|
||||||
|
overflow_correct_frequently: c_int,
|
||||||
|
) -> c_uint {
|
||||||
|
if state.is_null() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let state = unsafe { &mut *state };
|
||||||
|
if state.base.is_null()
|
||||||
|
|| state.dict_base.is_null()
|
||||||
|
|| state.dict_limit.is_null()
|
||||||
|
|| state.low_limit.is_null()
|
||||||
|
|| state.nb_overflow_corrections.is_null()
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let correction = window_correct_overflow(curr, cycle_log, max_dist);
|
||||||
|
let new_current = curr.wrapping_sub(correction);
|
||||||
|
let cycle_mask = 1u32.wrapping_shl(cycle_log).wrapping_sub(1);
|
||||||
|
debug_assert!(max_dist & max_dist.wrapping_sub(1) == 0);
|
||||||
|
debug_assert_eq!(curr & cycle_mask, new_current & cycle_mask);
|
||||||
|
debug_assert!(curr > new_current);
|
||||||
|
if overflow_correct_frequently == 0 {
|
||||||
|
debug_assert!(correction > 1 << 28);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
let base = &mut *state.base;
|
||||||
|
*base = (*base)
|
||||||
|
.cast::<u8>()
|
||||||
|
.wrapping_add(correction as usize)
|
||||||
|
.cast();
|
||||||
|
let dict_base = &mut *state.dict_base;
|
||||||
|
*dict_base = (*dict_base)
|
||||||
|
.cast::<u8>()
|
||||||
|
.wrapping_add(correction as usize)
|
||||||
|
.cast();
|
||||||
|
|
||||||
|
let limit_adjustment = correction.wrapping_add(ZSTD_WINDOW_START_INDEX);
|
||||||
|
let low_limit = &mut *state.low_limit;
|
||||||
|
if *low_limit < limit_adjustment {
|
||||||
|
*low_limit = ZSTD_WINDOW_START_INDEX;
|
||||||
|
} else {
|
||||||
|
*low_limit = (*low_limit).wrapping_sub(correction);
|
||||||
|
}
|
||||||
|
let dict_limit = &mut *state.dict_limit;
|
||||||
|
if *dict_limit < limit_adjustment {
|
||||||
|
*dict_limit = ZSTD_WINDOW_START_INDEX;
|
||||||
|
} else {
|
||||||
|
*dict_limit = (*dict_limit).wrapping_sub(correction);
|
||||||
|
}
|
||||||
|
|
||||||
|
debug_assert!(new_current >= max_dist);
|
||||||
|
debug_assert!(new_current.wrapping_sub(max_dist) >= ZSTD_WINDOW_START_INDEX);
|
||||||
|
debug_assert!(*low_limit <= new_current);
|
||||||
|
debug_assert!(*dict_limit <= new_current);
|
||||||
|
*state.nb_overflow_corrections = (*state.nb_overflow_corrections).wrapping_add(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
correction
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn window_can_overflow_correct(
|
fn window_can_overflow_correct(
|
||||||
curr: u32,
|
curr: u32,
|
||||||
@@ -16625,6 +16719,36 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn window_correction_rebases_projected_state() {
|
||||||
|
let buffer = [0u8; 512];
|
||||||
|
let initial_base = buffer.as_ptr().cast::<c_void>();
|
||||||
|
let initial_dict_base = unsafe { buffer.as_ptr().add(16).cast::<c_void>() };
|
||||||
|
let mut base = initial_base;
|
||||||
|
let mut dict_base = initial_dict_base;
|
||||||
|
let mut dict_limit = 0x110;
|
||||||
|
let mut low_limit = 0x120;
|
||||||
|
let mut nb_overflow_corrections = 2;
|
||||||
|
let mut state = ZSTD_rust_windowOverflowState {
|
||||||
|
base: &mut base,
|
||||||
|
dict_base: &mut dict_base,
|
||||||
|
dict_limit: &mut dict_limit,
|
||||||
|
low_limit: &mut low_limit,
|
||||||
|
nb_overflow_corrections: &mut nb_overflow_corrections,
|
||||||
|
};
|
||||||
|
|
||||||
|
let correction = unsafe {
|
||||||
|
ZSTD_rust_windowCorrectOverflowState(&mut state, 0x100, 3, 8, 1)
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(correction, 0xf0);
|
||||||
|
assert_eq!(base as usize, initial_base as usize + 0xf0);
|
||||||
|
assert_eq!(dict_base as usize, initial_dict_base as usize + 0xf0);
|
||||||
|
assert_eq!(dict_limit, 0x20);
|
||||||
|
assert_eq!(low_limit, 0x30);
|
||||||
|
assert_eq!(nb_overflow_corrections, 3);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn window_overflow_need_uses_the_explicit_current_max_boundary() {
|
fn window_overflow_need_uses_the_explicit_current_max_boundary() {
|
||||||
assert!(!window_need_overflow_correction(1024, false, 1024, false));
|
assert!(!window_need_overflow_correction(1024, false, 1024, false));
|
||||||
|
|||||||
Reference in New Issue
Block a user