feat(compress): move window dictionary invalidation to Rust
The window preparation callbacks already delegated dictionary-distance policy to Rust, but C still copied projected limits and cleared loaded-dictionary and match-state pointers. Add a checked pointer projection so Rust applies both window-limit and dictionary-validity side effects; C retains private window layout, pointer-index arithmetic, assertions, and diagnostics. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --tests (rust) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --tests -- -A clippy::manual-bits -D warnings (rust) - ulimit -v 41943040; make -j1 - ulimit -v 41943040; make -j1 -C tests test
This commit is contained in:
@@ -696,6 +696,18 @@ typedef char ZSTD_rust_window_dict_state_layout[
|
|||||||
&& offsetof(ZSTD_rust_windowDictState, loadedDictEnd) == 2 * sizeof(U32)
|
&& offsetof(ZSTD_rust_windowDictState, loadedDictEnd) == 2 * sizeof(U32)
|
||||||
&& offsetof(ZSTD_rust_windowDictState, invalidate) == 3 * sizeof(U32)
|
&& offsetof(ZSTD_rust_windowDictState, invalidate) == 3 * sizeof(U32)
|
||||||
&& sizeof(ZSTD_rust_windowDictState) == 4 * sizeof(U32)) ? 1 : -1];
|
&& sizeof(ZSTD_rust_windowDictState) == 4 * sizeof(U32)) ? 1 : -1];
|
||||||
|
typedef struct {
|
||||||
|
U32* lowLimit;
|
||||||
|
U32* dictLimit;
|
||||||
|
U32* loadedDictEnd;
|
||||||
|
const ZSTD_MatchState_t** dictMatchState;
|
||||||
|
} ZSTD_rust_windowDictPointers;
|
||||||
|
typedef char ZSTD_rust_window_dict_pointers_layout[
|
||||||
|
(offsetof(ZSTD_rust_windowDictPointers, lowLimit) == 0
|
||||||
|
&& offsetof(ZSTD_rust_windowDictPointers, dictLimit) == sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_windowDictPointers, loadedDictEnd) == 2 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_windowDictPointers, dictMatchState) == 3 * sizeof(void*)
|
||||||
|
&& sizeof(ZSTD_rust_windowDictPointers) == 4 * sizeof(void*)) ? 1 : -1];
|
||||||
void ZSTD_rust_windowInit(ZSTD_rust_windowUpdateState* state);
|
void ZSTD_rust_windowInit(ZSTD_rust_windowUpdateState* state);
|
||||||
U32 ZSTD_rust_windowIsEmpty(const void* nextSrc, const void* base,
|
U32 ZSTD_rust_windowIsEmpty(const void* nextSrc, const void* base,
|
||||||
U32 dictLimit, U32 lowLimit);
|
U32 dictLimit, U32 lowLimit);
|
||||||
@@ -704,6 +716,11 @@ ZSTD_rust_windowDictState ZSTD_rust_windowEnforceMaxDist(
|
|||||||
U32 blockEndIdx, U32 maxDist, ZSTD_rust_windowDictState state);
|
U32 blockEndIdx, U32 maxDist, ZSTD_rust_windowDictState state);
|
||||||
U32 ZSTD_rust_windowCheckDictValidity(U32 blockEndIdx, U32 maxDist,
|
U32 ZSTD_rust_windowCheckDictValidity(U32 blockEndIdx, U32 maxDist,
|
||||||
U32 loadedDictEnd, U32 dictLimit);
|
U32 loadedDictEnd, U32 dictLimit);
|
||||||
|
U32 ZSTD_rust_windowEnforceMaxDistState(
|
||||||
|
U32 blockEndIdx, U32 maxDist, ZSTD_rust_windowDictPointers* state);
|
||||||
|
U32 ZSTD_rust_windowCheckDictValidityState(
|
||||||
|
U32 blockEndIdx, U32 maxDist, U32 loadedDictEnd, U32 dictLimit,
|
||||||
|
U32* loadedDictEndPtr, const ZSTD_MatchState_t** dictMatchStatePtr);
|
||||||
U32 ZSTD_rust_windowUpdate(ZSTD_rust_windowUpdateState* state,
|
U32 ZSTD_rust_windowUpdate(ZSTD_rust_windowUpdateState* state,
|
||||||
const void* src, size_t srcSize,
|
const void* src, size_t srcSize,
|
||||||
int forceNonContiguous);
|
int forceNonContiguous);
|
||||||
@@ -1194,8 +1211,9 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
|
|||||||
U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
|
U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
|
||||||
U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
|
U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
|
||||||
U32 const oldDictLimit = window->dictLimit;
|
U32 const oldDictLimit = window->dictLimit;
|
||||||
ZSTD_rust_windowDictState state = {
|
ZSTD_rust_windowDictPointers state = {
|
||||||
window->lowLimit, window->dictLimit, loadedDictEnd, 0
|
&window->lowLimit, &window->dictLimit,
|
||||||
|
loadedDictEndPtr, dictMatchStatePtr
|
||||||
};
|
};
|
||||||
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
|
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
|
||||||
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
|
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
|
||||||
@@ -1213,17 +1231,11 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
|
|||||||
loadedDictEnd is expressed within the referential of the context,
|
loadedDictEnd is expressed within the referential of the context,
|
||||||
so it can be directly compared against blockEndIdx.
|
so it can be directly compared against blockEndIdx.
|
||||||
*/
|
*/
|
||||||
state = ZSTD_rust_windowEnforceMaxDist(blockEndIdx, maxDist, state);
|
if (ZSTD_rust_windowEnforceMaxDistState(blockEndIdx, maxDist, &state)) {
|
||||||
window->lowLimit = state.lowLimit;
|
|
||||||
window->dictLimit = state.dictLimit;
|
|
||||||
if (state.invalidate) {
|
|
||||||
if (oldDictLimit < window->lowLimit) {
|
if (oldDictLimit < window->lowLimit) {
|
||||||
DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
|
DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
|
||||||
(unsigned)oldDictLimit, (unsigned)window->lowLimit);
|
(unsigned)oldDictLimit, (unsigned)window->lowLimit);
|
||||||
}
|
}
|
||||||
/* On reaching window size, dictionaries are invalidated */
|
|
||||||
if (loadedDictEndPtr) *loadedDictEndPtr = state.loadedDictEnd;
|
|
||||||
if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1248,8 +1260,9 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
|
|||||||
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
|
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
|
||||||
assert(blockEndIdx >= loadedDictEnd);
|
assert(blockEndIdx >= loadedDictEnd);
|
||||||
|
|
||||||
if (ZSTD_rust_windowCheckDictValidity(
|
if (ZSTD_rust_windowCheckDictValidityState(
|
||||||
blockEndIdx, maxDist, loadedDictEnd, window->dictLimit)) {
|
blockEndIdx, maxDist, loadedDictEnd, window->dictLimit,
|
||||||
|
loadedDictEndPtr, dictMatchStatePtr)) {
|
||||||
/* On reaching window size, dictionaries are invalidated.
|
/* On reaching window size, dictionaries are invalidated.
|
||||||
* For simplification, if window size is reached anywhere within next block,
|
* For simplification, if window size is reached anywhere within next block,
|
||||||
* the dictionary is invalidated for the full block.
|
* the dictionary is invalidated for the full block.
|
||||||
@@ -1260,8 +1273,6 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
|
|||||||
* dictMatchState, so setting it to NULL is not a problem.
|
* dictMatchState, so setting it to NULL is not a problem.
|
||||||
*/
|
*/
|
||||||
DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
|
DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
|
||||||
*loadedDictEndPtr = 0;
|
|
||||||
*dictMatchStatePtr = NULL;
|
|
||||||
} else {
|
} else {
|
||||||
if (*loadedDictEndPtr != 0) {
|
if (*loadedDictEndPtr != 0) {
|
||||||
DEBUGLOG(6, "dictionary considered valid for current block");
|
DEBUGLOG(6, "dictionary considered valid for current block");
|
||||||
|
|||||||
+154
-4
@@ -10964,8 +10964,31 @@ const _: () = {
|
|||||||
assert!(size_of::<ZSTD_rust_windowDictState>() == 4 * size_of::<u32>());
|
assert!(size_of::<ZSTD_rust_windowDictState>() == 4 * size_of::<u32>());
|
||||||
};
|
};
|
||||||
|
|
||||||
#[no_mangle]
|
/// Pointers into the C-owned window and match-state fields used by dictionary
|
||||||
pub extern "C" fn ZSTD_rust_windowEnforceMaxDist(
|
/// invalidation. Rust owns the state transition; C retains the containing
|
||||||
|
/// private structs and computes source indices against their pointer layout.
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct ZSTD_rust_windowDictPointers {
|
||||||
|
low_limit: *mut c_uint,
|
||||||
|
dict_limit: *mut c_uint,
|
||||||
|
loaded_dict_end: *mut c_uint,
|
||||||
|
dict_match_state: *mut *const c_void,
|
||||||
|
}
|
||||||
|
|
||||||
|
const _: () = {
|
||||||
|
assert!(offset_of!(ZSTD_rust_windowDictPointers, low_limit) == 0);
|
||||||
|
assert!(offset_of!(ZSTD_rust_windowDictPointers, dict_limit) == size_of::<usize>());
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_windowDictPointers, loaded_dict_end) == 2 * size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_windowDictPointers, dict_match_state) == 3 * size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(size_of::<ZSTD_rust_windowDictPointers>() == 4 * size_of::<usize>());
|
||||||
|
};
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn window_enforce_max_dist(
|
||||||
block_end_idx: u32,
|
block_end_idx: u32,
|
||||||
max_dist: u32,
|
max_dist: u32,
|
||||||
mut state: ZSTD_rust_windowDictState,
|
mut state: ZSTD_rust_windowDictState,
|
||||||
@@ -10985,6 +11008,25 @@ pub extern "C" fn ZSTD_rust_windowEnforceMaxDist(
|
|||||||
state
|
state
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn window_check_dict_validity(
|
||||||
|
block_end_idx: u32,
|
||||||
|
max_dist: u32,
|
||||||
|
loaded_dict_end: u32,
|
||||||
|
dict_limit: u32,
|
||||||
|
) -> bool {
|
||||||
|
block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != dict_limit
|
||||||
|
}
|
||||||
|
|
||||||
|
#[no_mangle]
|
||||||
|
pub extern "C" fn ZSTD_rust_windowEnforceMaxDist(
|
||||||
|
block_end_idx: u32,
|
||||||
|
max_dist: u32,
|
||||||
|
state: ZSTD_rust_windowDictState,
|
||||||
|
) -> ZSTD_rust_windowDictState {
|
||||||
|
window_enforce_max_dist(block_end_idx, max_dist, state)
|
||||||
|
}
|
||||||
|
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub extern "C" fn ZSTD_rust_windowCheckDictValidity(
|
pub extern "C" fn ZSTD_rust_windowCheckDictValidity(
|
||||||
block_end_idx: u32,
|
block_end_idx: u32,
|
||||||
@@ -10992,8 +11034,75 @@ pub extern "C" fn ZSTD_rust_windowCheckDictValidity(
|
|||||||
loaded_dict_end: u32,
|
loaded_dict_end: u32,
|
||||||
dict_limit: u32,
|
dict_limit: u32,
|
||||||
) -> c_uint {
|
) -> c_uint {
|
||||||
(block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != dict_limit)
|
window_check_dict_validity(block_end_idx, max_dist, loaded_dict_end, dict_limit) as c_uint
|
||||||
as c_uint
|
}
|
||||||
|
|
||||||
|
/// Apply the window-limit transition to C-owned fields and invalidate any
|
||||||
|
/// dictionary pointers when the active window can no longer reference them.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_windowEnforceMaxDistState(
|
||||||
|
block_end_idx: u32,
|
||||||
|
max_dist: u32,
|
||||||
|
state: *mut ZSTD_rust_windowDictPointers,
|
||||||
|
) -> c_uint {
|
||||||
|
if state.is_null() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let state = unsafe { &mut *state };
|
||||||
|
if state.low_limit.is_null() || state.dict_limit.is_null() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let loaded_dict_end = if state.loaded_dict_end.is_null() {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
unsafe { *state.loaded_dict_end }
|
||||||
|
};
|
||||||
|
let next = window_enforce_max_dist(
|
||||||
|
block_end_idx,
|
||||||
|
max_dist,
|
||||||
|
ZSTD_rust_windowDictState {
|
||||||
|
lowLimit: unsafe { *state.low_limit },
|
||||||
|
dictLimit: unsafe { *state.dict_limit },
|
||||||
|
loadedDictEnd: loaded_dict_end,
|
||||||
|
invalidate: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
unsafe {
|
||||||
|
*state.low_limit = next.lowLimit;
|
||||||
|
*state.dict_limit = next.dictLimit;
|
||||||
|
if next.invalidate != 0 {
|
||||||
|
if !state.loaded_dict_end.is_null() {
|
||||||
|
*state.loaded_dict_end = next.loadedDictEnd;
|
||||||
|
}
|
||||||
|
if !state.dict_match_state.is_null() {
|
||||||
|
*state.dict_match_state = ptr::null();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next.invalidate
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check dictionary validity and apply the C-owned invalidation side effects.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_windowCheckDictValidityState(
|
||||||
|
block_end_idx: u32,
|
||||||
|
max_dist: u32,
|
||||||
|
loaded_dict_end: u32,
|
||||||
|
dict_limit: u32,
|
||||||
|
loaded_dict_end_ptr: *mut c_uint,
|
||||||
|
dict_match_state_ptr: *mut *const c_void,
|
||||||
|
) -> c_uint {
|
||||||
|
if loaded_dict_end_ptr.is_null() || dict_match_state_ptr.is_null() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let invalid = window_check_dict_validity(block_end_idx, max_dist, loaded_dict_end, dict_limit);
|
||||||
|
if invalid {
|
||||||
|
unsafe {
|
||||||
|
*loaded_dict_end_ptr = 0;
|
||||||
|
*dict_match_state_ptr = ptr::null();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
invalid as c_uint
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Update the rolling prefix/ext-dictionary window for one input segment.
|
/// Update the rolling prefix/ext-dictionary window for one input segment.
|
||||||
@@ -19308,6 +19417,47 @@ mod tests {
|
|||||||
assert_eq!(ZSTD_rust_windowCheckDictValidity(70, 64, 8, 10), 1);
|
assert_eq!(ZSTD_rust_windowCheckDictValidity(70, 64, 8, 10), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn window_dictionary_pointer_projection_applies_invalidation_side_effects() {
|
||||||
|
let mut low_limit = 2;
|
||||||
|
let mut dict_limit = 10;
|
||||||
|
let mut loaded_dict_end = 8;
|
||||||
|
let mut dict_match_state = ptr::dangling::<c_void>();
|
||||||
|
let mut state = ZSTD_rust_windowDictPointers {
|
||||||
|
low_limit: &mut low_limit,
|
||||||
|
dict_limit: &mut dict_limit,
|
||||||
|
loaded_dict_end: &mut loaded_dict_end,
|
||||||
|
dict_match_state: &mut dict_match_state,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
unsafe { ZSTD_rust_windowEnforceMaxDistState(100, 64, &mut state) },
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(low_limit, 36);
|
||||||
|
assert_eq!(dict_limit, 36);
|
||||||
|
assert_eq!(loaded_dict_end, 0);
|
||||||
|
assert!(dict_match_state.is_null());
|
||||||
|
|
||||||
|
loaded_dict_end = 8;
|
||||||
|
dict_match_state = ptr::dangling::<c_void>();
|
||||||
|
assert_eq!(
|
||||||
|
unsafe {
|
||||||
|
ZSTD_rust_windowCheckDictValidityState(
|
||||||
|
70,
|
||||||
|
64,
|
||||||
|
loaded_dict_end,
|
||||||
|
dict_limit,
|
||||||
|
&mut loaded_dict_end,
|
||||||
|
&mut dict_match_state,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(loaded_dict_end, 0);
|
||||||
|
assert!(dict_match_state.is_null());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn window_update_preserves_contiguous_input_and_clips_overlapping_extdict() {
|
fn window_update_preserves_contiguous_input_and_clips_overlapping_extdict() {
|
||||||
let storage = [0u8; 64];
|
let storage = [0u8; 64];
|
||||||
|
|||||||
Reference in New Issue
Block a user