ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move frame-block preparation order into Rust

Browse Source 
Move the frame-chunk match-state preparation policy into Rust while keeping
window and workspace mutations behind narrow C callbacks. Rust now guarantees
the original overflow-correction, dictionary-validity, maximum-distance, and
nextToUpdate-clamp order; the C side retains the private ZSTD_MatchState_t
layout and exact primitive operations.

Test Plan:
- cargo fmt --manifest-path rust/Cargo.toml -- --check
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml frame_chunk --lib (3 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (691 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && MAKEFLAGS=-j1 make -B -C programs -j1 zstd
- ulimit -v 41943040 && MAKEFLAGS=-j1 make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzz rounds passed)
This commit is contained in:
ddidderr
2026-07-19 19:01:15 +02:00
parent adaae03552
commit e95778be1a
3 changed files with 195 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+69 -14
View File
@@ -673,12 +673,38 @@ typedef char ZSTD_rust_reset_match_state_layout[
+ sizeof(void*))
? 1 : -1];
/* The frame-chunk loop is Rust-owned. Its callbacks keep the private
* ZSTD_CCtx and match-state layout in C: Rust only drives the block loop and
* passes this context back to these C-owned state-preparation/dispatch seams. */
typedef void (*ZSTD_rust_frameChunkPrepare_f)(void* context,
const void* src,
size_t blockSize);
/* The frame-chunk loop and preparation ordering are Rust-owned. These
* callbacks keep the private ZSTD_CCtx and match-state layout in C: Rust
* passes the context back to the narrow window/workspace operations. */
typedef void (*ZSTD_rust_frameChunkPrepareOverflow_f)(void* context,
const void* src,
size_t blockSize);
typedef void (*ZSTD_rust_frameChunkPrepareWindow_f)(void* context,
const void* src,
size_t blockSize,
U32 maxDist);
typedef void (*ZSTD_rust_frameChunkPrepareClamp_f)(void* context);
typedef struct {
void* callbackContext;
U32 maxDist;
ZSTD_rust_frameChunkPrepareOverflow_f correctOverflow;
ZSTD_rust_frameChunkPrepareWindow_f checkDictValidity;
ZSTD_rust_frameChunkPrepareWindow_f enforceMaxDist;
ZSTD_rust_frameChunkPrepareClamp_f clampNextToUpdate;
} ZSTD_rust_frameChunkPrepareState;
typedef char ZSTD_rust_frame_chunk_prepare_state_layout[
(offsetof(ZSTD_rust_frameChunkPrepareState, callbackContext) == 0
&& offsetof(ZSTD_rust_frameChunkPrepareState, maxDist) == sizeof(void*)
&& offsetof(ZSTD_rust_frameChunkPrepareState, correctOverflow)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_frameChunkPrepareState, checkDictValidity)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_frameChunkPrepareState, enforceMaxDist)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_frameChunkPrepareState, clampNextToUpdate)
== 5 * sizeof(void*)
&& sizeof(ZSTD_rust_frameChunkPrepareState) == 6 * sizeof(void*))
? 1 : -1];
typedef size_t (*ZSTD_rust_frameChunkCompress_f)(void* context,
void* dst,
size_t dstCapacity,
@@ -703,7 +729,7 @@ typedef struct {
int blockSplitterEnabled;
int checksumFlag;
int endingStage;
ZSTD_rust_frameChunkPrepare_f prepareBlock;
const ZSTD_rust_frameChunkPrepareState* prepareState;
ZSTD_rust_frameChunkCompress_f compressTarget;
ZSTD_rust_frameChunkCompress_f compressSplit;
ZSTD_rust_frameChunkCompress_f compressInternal;
@@ -735,7 +761,7 @@ typedef char ZSTD_rust_frame_chunk_state_layout[
== 7 * sizeof(void*) + sizeof(S64) + 4 * sizeof(int)
&& offsetof(ZSTD_rust_frameChunkState, endingStage)
== 7 * sizeof(void*) + sizeof(S64) + 5 * sizeof(int)
&& offsetof(ZSTD_rust_frameChunkState, prepareBlock)
&& offsetof(ZSTD_rust_frameChunkState, prepareState)
== 7 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int)
&& sizeof(ZSTD_rust_frameChunkState)
== 12 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int))
@@ -5053,24 +5079,46 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_MatchState_t* ms,
#include "zstd_preSplit.h"
static void ZSTD_rust_frameChunk_prepareBlock(void* context,
const void* src,
size_t blockSize)
static void ZSTD_rust_frameChunk_correctOverflow(void* context,
const void* src,
size_t blockSize)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
ZSTD_MatchState_t* const ms = &cctx->blockState.matchState;
U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
ZSTD_overflowCorrectIfNeeded(
ms, &cctx->workspace, &cctx->appliedParams,
src, (const BYTE*)src + blockSize);
}
static void ZSTD_rust_frameChunk_checkDictValidity(void* context,
const void* src,
size_t blockSize,
U32 maxDist)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
ZSTD_MatchState_t* const ms = &cctx->blockState.matchState;
ZSTD_checkDictValidity(
&ms->window, (const BYTE*)src + blockSize, maxDist,
&ms->loadedDictEnd, &ms->dictMatchState);
}
static void ZSTD_rust_frameChunk_enforceMaxDist(void* context,
const void* src,
size_t blockSize,
U32 maxDist)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
ZSTD_MatchState_t* const ms = &cctx->blockState.matchState;
ZSTD_window_enforceMaxDist(
&ms->window, src, maxDist,
&ms->loadedDictEnd, &ms->dictMatchState);
(void)blockSize;
}
static void ZSTD_rust_frameChunk_clampNextToUpdate(void* context)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
ZSTD_MatchState_t* const ms = &cctx->blockState.matchState;
/* Ensure hash/chain table insertion resumes no sooner than lowlimit. */
if (ms->nextToUpdate < ms->window.lowLimit)
ms->nextToUpdate = ms->window.lowLimit;
@@ -5121,6 +5169,13 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
U32 lastFrameChunk)
{
ZSTD_rust_frameChunkState state;
ZSTD_rust_frameChunkPrepareState prepareState;
prepareState.callbackContext = cctx;
prepareState.maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
prepareState.correctOverflow = ZSTD_rust_frameChunk_correctOverflow;
prepareState.checkDictValidity = ZSTD_rust_frameChunk_checkDictValidity;
prepareState.enforceMaxDist = ZSTD_rust_frameChunk_enforceMaxDist;
prepareState.clampNextToUpdate = ZSTD_rust_frameChunk_clampNextToUpdate;
state.callbackContext = cctx;
state.tmpWorkspace = cctx->tmpWorkspace;
state.checksumState = &cctx->xxhState;
@@ -5135,7 +5190,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
state.blockSplitterEnabled = ZSTD_blockSplitterEnabled(&cctx->appliedParams);
state.checksumFlag = cctx->appliedParams.fParams.checksumFlag;
state.endingStage = (int)ZSTDcs_ending;
state.prepareBlock = ZSTD_rust_frameChunk_prepareBlock;
state.prepareState = &prepareState;
state.compressTarget = ZSTD_rust_frameChunk_compressTarget;
state.compressSplit = ZSTD_rust_frameChunk_compressSplit;
state.compressInternal = ZSTD_rust_frameChunk_compressInternal;