ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move window dictionary invalidation to Rust

Browse Source 
The window preparation callbacks already delegated dictionary-distance policy to
Rust, but C still copied projected limits and cleared loaded-dictionary and
match-state pointers. Add a checked pointer projection so Rust applies both
window-limit and dictionary-validity side effects; C retains private window
layout, pointer-index arithmetic, assertions, and diagnostics.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --tests (rust)
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --tests -- -A clippy::manual-bits -D warnings (rust)
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test
This commit is contained in:
ddidderr
2026-07-22 00:34:27 +02:00
parent e3d1f7057d
commit 791b68cba7
2 changed files with 178 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress_internal.h
+24 -13
View File
@@ -696,6 +696,18 @@ typedef char ZSTD_rust_window_dict_state_layout[
&& offsetof(ZSTD_rust_windowDictState, loadedDictEnd) == 2 * sizeof(U32)
&& offsetof(ZSTD_rust_windowDictState, invalidate) == 3 * sizeof(U32)
&& sizeof(ZSTD_rust_windowDictState) == 4 * sizeof(U32)) ? 1 : -1];
typedef struct {
U32* lowLimit;
U32* dictLimit;
U32* loadedDictEnd;
const ZSTD_MatchState_t** dictMatchState;
} ZSTD_rust_windowDictPointers;
typedef char ZSTD_rust_window_dict_pointers_layout[
(offsetof(ZSTD_rust_windowDictPointers, lowLimit) == 0
&& offsetof(ZSTD_rust_windowDictPointers, dictLimit) == sizeof(void*)
&& offsetof(ZSTD_rust_windowDictPointers, loadedDictEnd) == 2 * sizeof(void*)
&& offsetof(ZSTD_rust_windowDictPointers, dictMatchState) == 3 * sizeof(void*)
&& sizeof(ZSTD_rust_windowDictPointers) == 4 * sizeof(void*)) ? 1 : -1];
void ZSTD_rust_windowInit(ZSTD_rust_windowUpdateState* state);
U32 ZSTD_rust_windowIsEmpty(const void* nextSrc, const void* base,
U32 dictLimit, U32 lowLimit);
@@ -704,6 +716,11 @@ ZSTD_rust_windowDictState ZSTD_rust_windowEnforceMaxDist(
U32 blockEndIdx, U32 maxDist, ZSTD_rust_windowDictState state);
U32 ZSTD_rust_windowCheckDictValidity(U32 blockEndIdx, U32 maxDist,
U32 loadedDictEnd, U32 dictLimit);
U32 ZSTD_rust_windowEnforceMaxDistState(
U32 blockEndIdx, U32 maxDist, ZSTD_rust_windowDictPointers* state);
U32 ZSTD_rust_windowCheckDictValidityState(
U32 blockEndIdx, U32 maxDist, U32 loadedDictEnd, U32 dictLimit,
U32* loadedDictEndPtr, const ZSTD_MatchState_t** dictMatchStatePtr);
U32 ZSTD_rust_windowUpdate(ZSTD_rust_windowUpdateState* state,
const void* src, size_t srcSize,
int forceNonContiguous);
@@ -1194,8 +1211,9 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
U32 const oldDictLimit = window->dictLimit;
ZSTD_rust_windowDictState state = {
window->lowLimit, window->dictLimit, loadedDictEnd, 0
ZSTD_rust_windowDictPointers state = {
&window->lowLimit, &window->dictLimit,
loadedDictEndPtr, dictMatchStatePtr
};
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
@@ -1213,17 +1231,11 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
loadedDictEnd is expressed within the referential of the context,
so it can be directly compared against blockEndIdx.
*/
state = ZSTD_rust_windowEnforceMaxDist(blockEndIdx, maxDist, state);
window->lowLimit = state.lowLimit;
window->dictLimit = state.dictLimit;
if (state.invalidate) {
if (ZSTD_rust_windowEnforceMaxDistState(blockEndIdx, maxDist, &state)) {
if (oldDictLimit < window->lowLimit) {
DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
(unsigned)oldDictLimit, (unsigned)window->lowLimit);
}
/* On reaching window size, dictionaries are invalidated */
if (loadedDictEndPtr) *loadedDictEndPtr = state.loadedDictEnd;
if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
}
}
@@ -1248,8 +1260,9 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
assert(blockEndIdx >= loadedDictEnd);
if (ZSTD_rust_windowCheckDictValidity(
blockEndIdx, maxDist, loadedDictEnd, window->dictLimit)) {
if (ZSTD_rust_windowCheckDictValidityState(
blockEndIdx, maxDist, loadedDictEnd, window->dictLimit,
loadedDictEndPtr, dictMatchStatePtr)) {
/* On reaching window size, dictionaries are invalidated.
* For simplification, if window size is reached anywhere within next block,
* the dictionary is invalidated for the full block.
@@ -1260,8 +1273,6 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
* dictMatchState, so setting it to NULL is not a problem.
*/
DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
*loadedDictEndPtr = 0;
*dictMatchStatePtr = NULL;
} else {
if (*loadedDictEndPtr != 0) {
DEBUGLOG(6, "dictionary considered valid for current block");