ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move window dictionary policy into Rust

Browse Source 
Move shared-window max-distance enforcement and dictionary-validity decisions into Rust. C keeps the private window pointer arithmetic, match-state pointer invalidation, and logging while applying the projected limit state returned by Rust.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1; cargo test --manifest-path rust/Cargo.toml
- ulimit -v 41943040; CARGO_BUILD_JOBS=1; cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
This commit is contained in:
ddidderr
2026-07-19 20:05:29 +02:00
parent f8cabd39a4
commit 2557319583
2 changed files with 121 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress_internal.h
+29 -8
View File
@@ -669,10 +669,26 @@ typedef char ZSTD_rust_window_update_state_layout[
&& offsetof(ZSTD_rust_windowUpdateState, dictLimit) == 3 * sizeof(void*)
&& offsetof(ZSTD_rust_windowUpdateState, lowLimit) == 3 * sizeof(void*) + sizeof(U32)
&& sizeof(ZSTD_rust_windowUpdateState) == 3 * sizeof(void*) + 2 * sizeof(U32)) ? 1 : -1];
typedef struct {
U32 lowLimit;
U32 dictLimit;
U32 loadedDictEnd;
U32 invalidate;
} ZSTD_rust_windowDictState;
typedef char ZSTD_rust_window_dict_state_layout[
(offsetof(ZSTD_rust_windowDictState, lowLimit) == 0
&& offsetof(ZSTD_rust_windowDictState, dictLimit) == sizeof(U32)
&& offsetof(ZSTD_rust_windowDictState, loadedDictEnd) == 2 * sizeof(U32)
&& offsetof(ZSTD_rust_windowDictState, invalidate) == 3 * sizeof(U32)
&& sizeof(ZSTD_rust_windowDictState) == 4 * sizeof(U32)) ? 1 : -1];
void ZSTD_rust_windowInit(ZSTD_rust_windowUpdateState* state);
U32 ZSTD_rust_windowIsEmpty(const void* nextSrc, const void* base,
U32 dictLimit, U32 lowLimit);
U32 ZSTD_rust_windowHasExtDict(U32 dictLimit, U32 lowLimit);
ZSTD_rust_windowDictState ZSTD_rust_windowEnforceMaxDist(
U32 blockEndIdx, U32 maxDist, ZSTD_rust_windowDictState state);
U32 ZSTD_rust_windowCheckDictValidity(U32 blockEndIdx, U32 maxDist,
U32 loadedDictEnd, U32 dictLimit);
U32 ZSTD_rust_windowUpdate(ZSTD_rust_windowUpdateState* state,
const void* src, size_t srcSize,
int forceNonContiguous);
@@ -1237,6 +1253,10 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
{
U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
U32 const oldDictLimit = window->dictLimit;
ZSTD_rust_windowDictState state = {
window->lowLimit, window->dictLimit, loadedDictEnd, 0
};
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
@@ -1253,16 +1273,16 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
loadedDictEnd is expressed within the referential of the context,
so it can be directly compared against blockEndIdx.
*/
if (blockEndIdx > maxDist + loadedDictEnd) {
U32 const newLowLimit = blockEndIdx - maxDist;
if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
if (window->dictLimit < window->lowLimit) {
state = ZSTD_rust_windowEnforceMaxDist(blockEndIdx, maxDist, state);
window->lowLimit = state.lowLimit;
window->dictLimit = state.dictLimit;
if (state.invalidate) {
if (oldDictLimit < window->lowLimit) {
DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
(unsigned)window->dictLimit, (unsigned)window->lowLimit);
window->dictLimit = window->lowLimit;
(unsigned)oldDictLimit, (unsigned)window->lowLimit);
}
/* On reaching window size, dictionaries are invalidated */
if (loadedDictEndPtr) *loadedDictEndPtr = 0;
if (loadedDictEndPtr) *loadedDictEndPtr = state.loadedDictEnd;
if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
}
}
@@ -1288,7 +1308,8 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
assert(blockEndIdx >= loadedDictEnd);
if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) {
if (ZSTD_rust_windowCheckDictValidity(
blockEndIdx, maxDist, loadedDictEnd, window->dictLimit)) {
/* On reaching window size, dictionaries are invalidated.
* For simplification, if window size is reached anywhere within next block,
* the dictionary is invalidated for the full block.
rust/src/zstd_compress.rs
+92
View File
@@ -6857,6 +6857,57 @@ pub extern "C" fn ZSTD_rust_windowHasExtDict(dict_limit: u32, low_limit: u32) ->
(low_limit < dict_limit) as c_uint
}
/// Project the dictionary/window limit policy while C retains pointer-owned
/// match-state invalidation and the private window layout.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTD_rust_windowDictState {
pub lowLimit: u32,
pub dictLimit: u32,
pub loadedDictEnd: u32,
pub invalidate: u32,
}
const _: () = {
assert!(offset_of!(ZSTD_rust_windowDictState, lowLimit) == 0);
assert!(offset_of!(ZSTD_rust_windowDictState, dictLimit) == size_of::<u32>());
assert!(offset_of!(ZSTD_rust_windowDictState, loadedDictEnd) == 2 * size_of::<u32>());
assert!(offset_of!(ZSTD_rust_windowDictState, invalidate) == 3 * size_of::<u32>());
assert!(size_of::<ZSTD_rust_windowDictState>() == 4 * size_of::<u32>());
};
#[no_mangle]
pub extern "C" fn ZSTD_rust_windowEnforceMaxDist(
block_end_idx: u32,
max_dist: u32,
mut state: ZSTD_rust_windowDictState,
) -> ZSTD_rust_windowDictState {
state.invalidate = 0;
if block_end_idx > max_dist.wrapping_add(state.loadedDictEnd) {
let new_low_limit = block_end_idx.wrapping_sub(max_dist);
if state.lowLimit < new_low_limit {
state.lowLimit = new_low_limit;
}
if state.dictLimit < state.lowLimit {
state.dictLimit = state.lowLimit;
}
state.loadedDictEnd = 0;
state.invalidate = 1;
}
state
}
#[no_mangle]
pub extern "C" fn ZSTD_rust_windowCheckDictValidity(
block_end_idx: u32,
max_dist: u32,
loaded_dict_end: u32,
dict_limit: u32,
) -> c_uint {
(block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != dict_limit)
as c_uint
}
/// Update the rolling prefix/ext-dictionary window for one input segment.
/// C owns the containing match state; Rust owns this five-field transition.
#[no_mangle]
@@ -12674,6 +12725,47 @@ mod tests {
assert_eq!(ZSTD_rust_windowHasExtDict(10, 10), 0);
}
#[test]
fn window_enforce_max_dist_updates_limits_and_invalidates_dictionary() {
let state = ZSTD_rust_windowEnforceMaxDist(
100,
64,
ZSTD_rust_windowDictState {
lowLimit: 2,
dictLimit: 10,
loadedDictEnd: 8,
invalidate: 0,
},
);
assert_eq!(state.lowLimit, 36);
assert_eq!(state.dictLimit, 36);
assert_eq!(state.loadedDictEnd, 0);
assert_eq!(state.invalidate, 1);
}
#[test]
fn window_dictionary_policy_preserves_valid_state_and_detects_mismatch() {
let state = ZSTD_rust_windowEnforceMaxDist(
70,
64,
ZSTD_rust_windowDictState {
lowLimit: 2,
dictLimit: 10,
loadedDictEnd: 8,
invalidate: 1,
},
);
assert_eq!(state.lowLimit, 2);
assert_eq!(state.dictLimit, 10);
assert_eq!(state.loadedDictEnd, 8);
assert_eq!(state.invalidate, 0);
assert_eq!(ZSTD_rust_windowCheckDictValidity(70, 64, 8, 8), 0);
assert_eq!(ZSTD_rust_windowCheckDictValidity(100, 64, 8, 8), 1);
assert_eq!(ZSTD_rust_windowCheckDictValidity(70, 64, 8, 10), 1);
}
#[test]
fn window_update_preserves_contiguous_input_and_clips_overlapping_extdict() {
let storage = [0u8; 64];