ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move match-state reset policy into Rust

Browse Source 
Move the reset decision tree for match-state tables from C into a Rust-owned
callback projection. Rust now decides chain/hash3/row/optimal allocations,
reset versus continue behavior, clean versus leave-dirty policy, scalar
publication, and allocation-error propagation. C retains the cwksp phases,
window invalidation, private pointer assignments, zeroing, and hash-salt
implementation behind callbacks, so the private match-state and workspace
layouts remain C-owned.

Update the migration boundary documentation and add callback-order probes for
CCtx optimal storage, CCtx row salting, CDict row zeroing, and table allocation
failure.

Test Plan:
- cargo fmt --manifest-path rust/Cargo.toml -- --check
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (677 passed)
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzzer rounds)
This commit is contained in:
ddidderr
2026-07-19 17:43:45 +02:00
parent 09334d2065
commit 127dd21195
3 changed files with 934 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+234 -76
View File
@@ -614,6 +614,65 @@ U64 ZSTD_rust_advanceHashSalt(U64 hashSalt, U64 hashSaltEntropy);
int ZSTD_rust_indexTooCloseToMax(size_t nextSrcBaseOffset);
int ZSTD_rust_dictTooBig(size_t loadedDictSize);
/* Match-state reset orchestration lives in Rust. The callbacks keep the
* workspace implementation, window representation, and private pointer
* fields in C. */
typedef void (*ZSTD_rust_resetMatchStateCallback_f)(void* context);
typedef void (*ZSTD_rust_resetMatchStateSetPointer_f)(
void* context, int pointerKind, void* pointer);
typedef void* (*ZSTD_rust_resetMatchStateReserve_f)(
void* context, int reserveKind, size_t size);
typedef int (*ZSTD_rust_resetMatchStateReserveFailed_f)(void* context);
typedef void (*ZSTD_rust_resetMatchStateZero_f)(
void* context, void* pointer, size_t size);
typedef struct {
void* callbackContext;
ZSTD_compressionParameters cParams;
int dedicatedDictSearch;
int useRowMatchFinder;
int compResetPolicy;
int indexResetPolicy;
int resetTarget;
unsigned hashLog3Max;
size_t litFreqSize;
size_t litLengthFreqSize;
size_t matchLengthFreqSize;
size_t offCodeFreqSize;
size_t matchSize;
size_t optimalSize;
void* hashLog3;
void* rowHashLog;
void* hashSalt;
void* lazySkipping;
void* cParamsOut;
ZSTD_rust_resetMatchStateSetPointer_f setPointer;
ZSTD_rust_resetMatchStateCallback_f windowInit;
ZSTD_rust_resetMatchStateCallback_f markTablesDirty;
ZSTD_rust_resetMatchStateCallback_f invalidateMatchState;
ZSTD_rust_resetMatchStateCallback_f clearTables;
ZSTD_rust_resetMatchStateCallback_f cleanTables;
ZSTD_rust_resetMatchStateCallback_f advanceHashSalt;
ZSTD_rust_resetMatchStateReserve_f reserve;
ZSTD_rust_resetMatchStateReserveFailed_f reserveFailed;
ZSTD_rust_resetMatchStateZero_f zero;
} ZSTD_rust_resetMatchStateState;
size_t ZSTD_rust_resetMatchState(const ZSTD_rust_resetMatchStateState* state);
typedef char ZSTD_rust_reset_match_state_layout[
(offsetof(ZSTD_rust_resetMatchStateState, callbackContext) == 0
&& offsetof(ZSTD_rust_resetMatchStateState, cParams) == sizeof(void*)
&& offsetof(ZSTD_rust_resetMatchStateState, litFreqSize)
> offsetof(ZSTD_rust_resetMatchStateState, hashLog3Max)
&& offsetof(ZSTD_rust_resetMatchStateState, hashLog3)
== offsetof(ZSTD_rust_resetMatchStateState, optimalSize)
+ sizeof(size_t)
&& offsetof(ZSTD_rust_resetMatchStateState, setPointer)
== offsetof(ZSTD_rust_resetMatchStateState, cParamsOut)
+ sizeof(void*)
&& sizeof(ZSTD_rust_resetMatchStateState)
== offsetof(ZSTD_rust_resetMatchStateState, zero)
+ sizeof(void*))
? 1 : -1];
/* The frame-chunk loop is Rust-owned. Its callbacks keep the private
* ZSTD_CCtx and match-state layout in C: Rust only drives the block loop and
* passes this context back to these C-owned state-preparation/dispatch seams. */
@@ -3308,6 +3367,147 @@ static void ZSTD_advanceHashSalt(ZSTD_MatchState_t* ms) {
ms->hashSalt = ZSTD_rust_advanceHashSalt(ms->hashSalt, (U64)ms->hashSaltEntropy);
}
typedef struct {
ZSTD_MatchState_t* ms;
ZSTD_cwksp* ws;
} ZSTD_rust_resetMatchStateContext;
enum {
ZSTD_RUST_RESET_MATCH_RESERVE_TABLE = 0,
ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE = 1,
ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED64 = 2
};
enum {
ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE = 0,
ZSTD_RUST_RESET_MATCH_POINTER_CHAIN_TABLE = 1,
ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE3 = 2,
ZSTD_RUST_RESET_MATCH_POINTER_TAG_TABLE = 3,
ZSTD_RUST_RESET_MATCH_POINTER_LIT_FREQ = 4,
ZSTD_RUST_RESET_MATCH_POINTER_LIT_LENGTH_FREQ = 5,
ZSTD_RUST_RESET_MATCH_POINTER_MATCH_LENGTH_FREQ = 6,
ZSTD_RUST_RESET_MATCH_POINTER_OFF_CODE_FREQ = 7,
ZSTD_RUST_RESET_MATCH_POINTER_MATCH_TABLE = 8,
ZSTD_RUST_RESET_MATCH_POINTER_PRICE_TABLE = 9
};
static void ZSTD_rust_resetMatchState_windowInit(void* opaque)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
ZSTD_window_init(&context->ms->window);
}
static void ZSTD_rust_resetMatchState_markTablesDirty(void* opaque)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
ZSTD_cwksp_mark_tables_dirty(context->ws);
}
static void ZSTD_rust_resetMatchState_invalidate(void* opaque)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
ZSTD_invalidateMatchState(context->ms);
}
static void ZSTD_rust_resetMatchState_clearTables(void* opaque)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
ZSTD_cwksp_clear_tables(context->ws);
}
static void ZSTD_rust_resetMatchState_cleanTables(void* opaque)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
ZSTD_cwksp_clean_tables(context->ws);
}
static void ZSTD_rust_resetMatchState_advanceHashSalt(void* opaque)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
ZSTD_advanceHashSalt(context->ms);
}
static void* ZSTD_rust_resetMatchState_reserve(
void* opaque, int reserveKind, size_t size)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
switch (reserveKind) {
case ZSTD_RUST_RESET_MATCH_RESERVE_TABLE:
return ZSTD_cwksp_reserve_table(context->ws, size);
case ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE:
return ZSTD_cwksp_reserve_aligned_init_once(context->ws, size);
case ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED64:
return ZSTD_cwksp_reserve_aligned64(context->ws, size);
default:
assert(0);
return NULL;
}
}
static int ZSTD_rust_resetMatchState_reserveFailed(void* opaque)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
return ZSTD_cwksp_reserve_failed(context->ws);
}
static void ZSTD_rust_resetMatchState_setPointer(
void* opaque, int pointerKind, void* pointer)
{
ZSTD_rust_resetMatchStateContext* const context =
(ZSTD_rust_resetMatchStateContext*)opaque;
ZSTD_MatchState_t* const ms = context->ms;
switch (pointerKind) {
case ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE:
ms->hashTable = (U32*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_CHAIN_TABLE:
ms->chainTable = (U32*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE3:
ms->hashTable3 = (U32*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_TAG_TABLE:
ms->tagTable = (BYTE*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_LIT_FREQ:
ms->opt.litFreq = (unsigned*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_LIT_LENGTH_FREQ:
ms->opt.litLengthFreq = (unsigned*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_MATCH_LENGTH_FREQ:
ms->opt.matchLengthFreq = (unsigned*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_OFF_CODE_FREQ:
ms->opt.offCodeFreq = (unsigned*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_MATCH_TABLE:
ms->opt.matchTable = (ZSTD_match_t*)pointer;
break;
case ZSTD_RUST_RESET_MATCH_POINTER_PRICE_TABLE:
ms->opt.priceTable = (ZSTD_optimal_t*)pointer;
break;
default:
assert(0);
break;
}
}
static void ZSTD_rust_resetMatchState_zero(
void* opaque, void* pointer, size_t size)
{
(void)opaque;
ZSTD_memset(pointer, 0, size);
}
static size_t
ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
ZSTD_cwksp* ws,
@@ -3317,82 +3517,40 @@ ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
const ZSTD_indexResetPolicy_e forceResetIndex,
const ZSTD_resetTarget_e forWho)
{
/* disable chain table allocation for fast or row-based strategies */
size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder,
ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict))
? ((size_t)1 << cParams->chainLog)
: 0;
size_t const hSize = ((size_t)1) << cParams->hashLog;
U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
assert(useRowMatchFinder != ZSTD_ps_auto);
if (forceResetIndex == ZSTDirp_reset) {
ZSTD_window_init(&ms->window);
ZSTD_cwksp_mark_tables_dirty(ws);
}
ms->hashLog3 = hashLog3;
ms->lazySkipping = 0;
ZSTD_invalidateMatchState(ms);
assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */
ZSTD_cwksp_clear_tables(ws);
DEBUGLOG(5, "reserving table space");
/* table Space */
ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32));
ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32));
ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32));
RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
"failed a workspace allocation in ZSTD_reset_matchState");
DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty);
if (crp!=ZSTDcrp_leaveDirty) {
/* reset tables only */
ZSTD_cwksp_clean_tables(ws);
}
if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
/* Row match finder needs an additional table of hashes ("tags") */
size_t const tagTableSize = hSize;
/* We want to generate a new salt in case we reset a Cctx, but we always want to use
* 0 when we reset a Cdict */
if(forWho == ZSTD_resetTarget_CCtx) {
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize);
ZSTD_advanceHashSalt(ms);
} else {
/* When we are not salting we want to always memset the memory */
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize);
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ms->hashSalt = 0;
}
{ /* Switch to 32-entry rows if searchLog is 5 (or more) */
U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
assert(cParams->hashLog >= rowLog);
ms->rowHashLog = cParams->hashLog - rowLog;
}
}
/* opt parser space */
if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
DEBUGLOG(4, "reserving optimal parser space");
ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<<Litbits) * sizeof(unsigned));
ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned));
ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned));
ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned));
ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t));
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t));
}
ms->cParams = *cParams;
RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
"failed a workspace allocation in ZSTD_reset_matchState");
return 0;
ZSTD_rust_resetMatchStateContext context;
ZSTD_rust_resetMatchStateState state;
context.ms = ms;
context.ws = ws;
state.callbackContext = &context;
state.cParams = *cParams;
state.dedicatedDictSearch = ms->dedicatedDictSearch;
state.useRowMatchFinder = (int)useRowMatchFinder;
state.compResetPolicy = (int)crp;
state.indexResetPolicy = (int)forceResetIndex;
state.resetTarget = (int)forWho;
state.hashLog3Max = ZSTD_HASHLOG3_MAX;
state.litFreqSize = (1 << Litbits) * sizeof(unsigned);
state.litLengthFreqSize = (MaxLL + 1) * sizeof(unsigned);
state.matchLengthFreqSize = (MaxML + 1) * sizeof(unsigned);
state.offCodeFreqSize = (MaxOff + 1) * sizeof(unsigned);
state.matchSize = ZSTD_OPT_SIZE * sizeof(ZSTD_match_t);
state.optimalSize = ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t);
state.hashLog3 = &ms->hashLog3;
state.rowHashLog = &ms->rowHashLog;
state.hashSalt = &ms->hashSalt;
state.lazySkipping = &ms->lazySkipping;
state.cParamsOut = &ms->cParams;
state.setPointer = ZSTD_rust_resetMatchState_setPointer;
state.windowInit = ZSTD_rust_resetMatchState_windowInit;
state.markTablesDirty = ZSTD_rust_resetMatchState_markTablesDirty;
state.invalidateMatchState = ZSTD_rust_resetMatchState_invalidate;
state.clearTables = ZSTD_rust_resetMatchState_clearTables;
state.cleanTables = ZSTD_rust_resetMatchState_cleanTables;
state.advanceHashSalt = ZSTD_rust_resetMatchState_advanceHashSalt;
state.reserve = ZSTD_rust_resetMatchState_reserve;
state.reserveFailed = ZSTD_rust_resetMatchState_reserveFailed;
state.zero = ZSTD_rust_resetMatchState_zero;
return ZSTD_rust_resetMatchState(&state);
}
/* ZSTD_indexTooCloseToMax() :
rust/README.md
+4 -4
View File
@@ -166,10 +166,10 @@ CDict advanced private workspace construction, private static-CCtx and
static-CDict workspace construction and dictionary-content allocation/loading,
and advanced-CDict dictionary-content loading remain in C. Rust now owns
advanced-CDict custom-memory validation, workspace-size query/allocation,
allocation/create/init cleanup ordering, and the CCtx workspace-size formula;
C retains private layout-size inputs, workspace layout, and allocator
callbacks. Reset policy,
private CCtx/matchfinder/workspace operations, and codec/adaptive-policy
allocation/create/init cleanup ordering, the CCtx workspace-size formula, and
the match-state reset policy/order; C retains private layout-size inputs,
workspace layout, and allocator callbacks. Private CCtx reset/matchfinder/
workspace operations and codec/adaptive-policy
callbacks remain in C. CDict initialization ordering and scalar publication,
shared compression-begin dictionary selection, CDict reset attach-versus-copy
selection, and CDict-begin parameter selection, initialization ordering, and
rust/src/zstd_compress.rs
+696 -3
View File
@@ -24,11 +24,12 @@ use crate::zstd_compress_frame::{
use crate::zstd_compress_literals::min_gain;
use crate::zstd_compress_params::{
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters, ZSTD_rustMatchStateSizing,
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_checkCParams, ZSTD_rust_params_defaultCLevel,
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_allocateChainTable,
ZSTD_rust_params_checkCParams, ZSTD_rust_params_defaultCLevel,
ZSTD_rust_params_estimateMatchStateSize, ZSTD_rust_params_getParamsInternal,
ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_resolveMaxBlockSize,
ZSTD_rust_params_selectCParams, ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO,
ZSTD_RUST_PS_DISABLE, ZSTD_RUST_PS_ENABLE,
ZSTD_rust_params_rowMatchFinderUsed, ZSTD_rust_params_selectCParams,
ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO, ZSTD_RUST_PS_DISABLE, ZSTD_RUST_PS_ENABLE,
};
use crate::zstd_compress_params_api::{
ZSTD_CCtxParams_setParameter, ZSTD_CCtx_params, ZSTD_rust_isUpdateAuthorized,
@@ -5221,6 +5222,356 @@ pub extern "C" fn ZSTD_rust_dictTooBig(loaded_dict_size: usize) -> c_int {
dict_too_big(loaded_dict_size) as c_int
}
type ResetMatchStateCallback = unsafe extern "C" fn(*mut c_void);
type ResetMatchStateSetPointer = unsafe extern "C" fn(*mut c_void, c_int, *mut c_void);
type ResetMatchStateReserve = unsafe extern "C" fn(*mut c_void, c_int, usize) -> *mut c_void;
type ResetMatchStateReserveFailed = unsafe extern "C" fn(*mut c_void) -> c_int;
type ResetMatchStateZero = unsafe extern "C" fn(*mut c_void, *mut c_void, usize);
const RESET_MATCH_RESERVE_TABLE: c_int = 0;
const RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE: c_int = 1;
const RESET_MATCH_RESERVE_ALIGNED64: c_int = 2;
const RESET_MATCH_POINTER_HASH_TABLE: c_int = 0;
const RESET_MATCH_POINTER_CHAIN_TABLE: c_int = 1;
const RESET_MATCH_POINTER_HASH_TABLE3: c_int = 2;
const RESET_MATCH_POINTER_TAG_TABLE: c_int = 3;
const RESET_MATCH_POINTER_LIT_FREQ: c_int = 4;
const RESET_MATCH_POINTER_LIT_LENGTH_FREQ: c_int = 5;
const RESET_MATCH_POINTER_MATCH_LENGTH_FREQ: c_int = 6;
const RESET_MATCH_POINTER_OFF_CODE_FREQ: c_int = 7;
const RESET_MATCH_POINTER_MATCH_TABLE: c_int = 8;
const RESET_MATCH_POINTER_PRICE_TABLE: c_int = 9;
/// Private C layout pointers and callback seams for match-state reset.
///
/// Rust owns the reset policy and allocation order. C retains the workspace,
/// window, and private match-state pointer representations behind callbacks.
#[repr(C)]
pub struct ZSTD_rust_resetMatchStateState {
callback_context: *mut c_void,
c_params: ZSTD_compressionParameters,
dedicated_dict_search: c_int,
use_row_match_finder: c_int,
comp_reset_policy: c_int,
index_reset_policy: c_int,
reset_target: c_int,
hash_log3_max: c_uint,
lit_freq_size: usize,
lit_length_freq_size: usize,
match_length_freq_size: usize,
off_code_freq_size: usize,
match_size: usize,
optimal_size: usize,
hash_log3: *mut c_uint,
row_hash_log: *mut c_uint,
hash_salt: *mut u64,
lazy_skipping: *mut c_int,
c_params_out: *mut ZSTD_compressionParameters,
set_pointer: Option<ResetMatchStateSetPointer>,
window_init: Option<ResetMatchStateCallback>,
mark_tables_dirty: Option<ResetMatchStateCallback>,
invalidate_match_state: Option<ResetMatchStateCallback>,
clear_tables: Option<ResetMatchStateCallback>,
clean_tables: Option<ResetMatchStateCallback>,
advance_hash_salt: Option<ResetMatchStateCallback>,
reserve: Option<ResetMatchStateReserve>,
reserve_failed: Option<ResetMatchStateReserveFailed>,
zero: Option<ResetMatchStateZero>,
}
const _: () = {
assert!(size_of::<ResetMatchStateCallback>() == size_of::<usize>());
assert!(size_of::<ResetMatchStateSetPointer>() == size_of::<usize>());
assert!(size_of::<ResetMatchStateReserve>() == size_of::<usize>());
assert!(size_of::<ResetMatchStateReserveFailed>() == size_of::<usize>());
assert!(size_of::<ResetMatchStateZero>() == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_resetMatchStateState, callback_context) == 0);
assert!(offset_of!(ZSTD_rust_resetMatchStateState, c_params) == size_of::<usize>());
assert!(
offset_of!(ZSTD_rust_resetMatchStateState, lit_freq_size)
> offset_of!(ZSTD_rust_resetMatchStateState, hash_log3_max)
);
assert!(
offset_of!(ZSTD_rust_resetMatchStateState, hash_log3)
== offset_of!(ZSTD_rust_resetMatchStateState, optimal_size) + size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_resetMatchStateState, set_pointer)
== offset_of!(ZSTD_rust_resetMatchStateState, c_params_out) + size_of::<usize>()
);
assert!(
size_of::<ZSTD_rust_resetMatchStateState>()
== offset_of!(ZSTD_rust_resetMatchStateState, zero) + size_of::<usize>()
);
};
#[inline]
fn reset_match_state_table_size(log: u32) -> Option<usize> {
1usize.checked_shl(log)
}
/// Reset and allocate one private match-state projection.
///
/// The callback context deliberately remains opaque here. This keeps C's
/// workspace phases and match-state layout out of the Rust ABI while making
/// the policy/order independently testable.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_resetMatchState(
state: *const ZSTD_rust_resetMatchStateState,
) -> usize {
if state.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let state = unsafe { &*state };
let Some(set_pointer) = state.set_pointer else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(window_init) = state.window_init else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(mark_tables_dirty) = state.mark_tables_dirty else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(invalidate_match_state) = state.invalidate_match_state else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(clear_tables) = state.clear_tables else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(clean_tables) = state.clean_tables else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(advance_hash_salt) = state.advance_hash_salt else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(reserve) = state.reserve else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(reserve_failed) = state.reserve_failed else {
return ERROR(ZstdErrorCode::Generic);
};
let Some(zero) = state.zero else {
return ERROR(ZstdErrorCode::Generic);
};
if state.callback_context.is_null()
|| state.hash_log3.is_null()
|| state.row_hash_log.is_null()
|| state.hash_salt.is_null()
|| state.lazy_skipping.is_null()
|| state.c_params_out.is_null()
{
return ERROR(ZstdErrorCode::Generic);
}
if state.use_row_match_finder == ZSTD_RUST_PS_AUTO
|| !matches!(state.comp_reset_policy, 0..=1)
|| !matches!(state.index_reset_policy, 0..=1)
|| !matches!(state.reset_target, 0..=1)
{
return ERROR(ZstdErrorCode::Generic);
}
let chain_size = if ZSTD_rust_params_allocateChainTable(
state.c_params.strategy,
state.use_row_match_finder,
c_int::from(state.dedicated_dict_search != 0 && state.reset_target == 0),
) != 0
{
let Some(size) = reset_match_state_table_size(state.c_params.chainLog) else {
return ERROR(ZstdErrorCode::Generic);
};
size
} else {
0
};
let Some(hash_size) = reset_match_state_table_size(state.c_params.hashLog) else {
return ERROR(ZstdErrorCode::Generic);
};
let hash_log3 = if state.reset_target == 1 && state.c_params.minMatch == 3 {
state.hash_log3_max.min(state.c_params.windowLog)
} else {
0
};
let hash3_size = if hash_log3 == 0 {
0
} else {
let Some(size) = reset_match_state_table_size(hash_log3) else {
return ERROR(ZstdErrorCode::Generic);
};
size
};
if state.index_reset_policy == 1 {
unsafe {
window_init(state.callback_context);
mark_tables_dirty(state.callback_context);
}
}
unsafe {
*state.hash_log3 = hash_log3;
*state.lazy_skipping = 0;
invalidate_match_state(state.callback_context);
debug_assert_eq!(reserve_failed(state.callback_context), 0);
clear_tables(state.callback_context);
}
unsafe {
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_HASH_TABLE,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_TABLE,
hash_size.wrapping_mul(size_of::<c_uint>()),
),
);
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_CHAIN_TABLE,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_TABLE,
chain_size.wrapping_mul(size_of::<c_uint>()),
),
);
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_HASH_TABLE3,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_TABLE,
hash3_size.wrapping_mul(size_of::<c_uint>()),
),
);
}
if unsafe { reserve_failed(state.callback_context) } != 0 {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
if state.comp_reset_policy != 1 {
unsafe { clean_tables(state.callback_context) };
}
if ZSTD_rust_params_rowMatchFinderUsed(state.c_params.strategy, state.use_row_match_finder) != 0
{
if state.reset_target == 1 {
let tag_table = unsafe {
reserve(
state.callback_context,
RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE,
hash_size,
)
};
unsafe {
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_TAG_TABLE,
tag_table,
);
advance_hash_salt(state.callback_context);
}
} else {
let tag_table = unsafe {
reserve(
state.callback_context,
RESET_MATCH_RESERVE_ALIGNED64,
hash_size,
)
};
unsafe {
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_TAG_TABLE,
tag_table,
);
}
if unsafe { reserve_failed(state.callback_context) } != 0 {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
unsafe {
zero(state.callback_context, tag_table, hash_size);
*state.hash_salt = 0;
}
}
if unsafe { reserve_failed(state.callback_context) } != 0 {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
let row_log = state.c_params.searchLog.clamp(4, 6);
debug_assert!(state.c_params.hashLog >= row_log);
unsafe {
*state.row_hash_log = state.c_params.hashLog.wrapping_sub(row_log);
}
}
if state.reset_target == 1 && state.c_params.strategy >= 7 {
unsafe {
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_LIT_FREQ,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_ALIGNED64,
state.lit_freq_size,
),
);
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_LIT_LENGTH_FREQ,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_ALIGNED64,
state.lit_length_freq_size,
),
);
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_MATCH_LENGTH_FREQ,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_ALIGNED64,
state.match_length_freq_size,
),
);
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_OFF_CODE_FREQ,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_ALIGNED64,
state.off_code_freq_size,
),
);
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_MATCH_TABLE,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_ALIGNED64,
state.match_size,
),
);
set_pointer(
state.callback_context,
RESET_MATCH_POINTER_PRICE_TABLE,
reserve(
state.callback_context,
RESET_MATCH_RESERVE_ALIGNED64,
state.optimal_size,
),
);
}
}
unsafe {
*state.c_params_out = state.c_params;
}
if unsafe { reserve_failed(state.callback_context) } != 0 {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
0
}
#[inline]
fn sizeof_local_dict(dict_buffer_present: c_int, dict_size: usize, cdict_size: usize) -> usize {
let buffer_size = if dict_buffer_present != 0 {
@@ -6821,6 +7172,348 @@ mod tests {
const ZSTD_BTOPT: c_int = 7;
const ZSTD_BTULTRA2: c_int = 9;
#[derive(Default)]
struct ResetMatchStateTestContext {
events: Vec<c_int>,
reserves: Vec<(c_int, usize)>,
pointers: Vec<(c_int, *mut c_void)>,
zeroed: Vec<(usize, usize)>,
storage: [usize; 32],
fail_after: usize,
}
unsafe fn reset_match_state_test_context(
context: *mut c_void,
) -> &'static mut ResetMatchStateTestContext {
unsafe { &mut *context.cast::<ResetMatchStateTestContext>() }
}
unsafe extern "C" fn reset_match_state_test_callback(context: *mut c_void) {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(1);
}
unsafe extern "C" fn reset_match_state_test_mark_dirty(context: *mut c_void) {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(2);
}
unsafe extern "C" fn reset_match_state_test_invalidate(context: *mut c_void) {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(3);
}
unsafe extern "C" fn reset_match_state_test_clear(context: *mut c_void) {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(4);
}
unsafe extern "C" fn reset_match_state_test_clean(context: *mut c_void) {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(5);
}
unsafe extern "C" fn reset_match_state_test_advance(context: *mut c_void) {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(6);
}
unsafe extern "C" fn reset_match_state_test_reserve(
context: *mut c_void,
reserve_kind: c_int,
size: usize,
) -> *mut c_void {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(100 + reserve_kind);
let index = context.reserves.len();
context.reserves.push((reserve_kind, size));
if context.fail_after != 0 && index + 1 >= context.fail_after {
ptr::null_mut()
} else {
unsafe { context.storage.as_mut_ptr().add(index).cast() }
}
}
unsafe extern "C" fn reset_match_state_test_reserve_failed(context: *mut c_void) -> c_int {
let context = unsafe { reset_match_state_test_context(context) };
c_int::from(context.fail_after != 0 && context.reserves.len() >= context.fail_after)
}
unsafe extern "C" fn reset_match_state_test_set_pointer(
context: *mut c_void,
pointer_kind: c_int,
pointer: *mut c_void,
) {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(200 + pointer_kind);
context.pointers.push((pointer_kind, pointer));
}
unsafe extern "C" fn reset_match_state_test_zero(
context: *mut c_void,
pointer: *mut c_void,
size: usize,
) {
let context = unsafe { reset_match_state_test_context(context) };
context.events.push(7);
context.zeroed.push((pointer as usize, size));
}
#[allow(clippy::too_many_arguments)]
fn reset_match_state_test_state(
context: &mut ResetMatchStateTestContext,
c_params: ZSTD_compressionParameters,
dedicated_dict_search: c_int,
use_row_match_finder: c_int,
comp_reset_policy: c_int,
index_reset_policy: c_int,
reset_target: c_int,
hash_log3: &mut c_uint,
row_hash_log: &mut c_uint,
hash_salt: &mut u64,
lazy_skipping: &mut c_int,
c_params_out: &mut ZSTD_compressionParameters,
) -> ZSTD_rust_resetMatchStateState {
ZSTD_rust_resetMatchStateState {
callback_context: (context as *mut ResetMatchStateTestContext).cast(),
c_params,
dedicated_dict_search,
use_row_match_finder,
comp_reset_policy,
index_reset_policy,
reset_target,
hash_log3_max: 17,
lit_freq_size: 256 * size_of::<c_uint>(),
lit_length_freq_size: 36 * size_of::<c_uint>(),
match_length_freq_size: 53 * size_of::<c_uint>(),
off_code_freq_size: 32 * size_of::<c_uint>(),
match_size: 4099 * 2 * size_of::<c_uint>(),
optimal_size: 4099 * 8 * size_of::<c_uint>(),
hash_log3,
row_hash_log,
hash_salt,
lazy_skipping,
c_params_out,
set_pointer: Some(reset_match_state_test_set_pointer),
window_init: Some(reset_match_state_test_callback),
mark_tables_dirty: Some(reset_match_state_test_mark_dirty),
invalidate_match_state: Some(reset_match_state_test_invalidate),
clear_tables: Some(reset_match_state_test_clear),
clean_tables: Some(reset_match_state_test_clean),
advance_hash_salt: Some(reset_match_state_test_advance),
reserve: Some(reset_match_state_test_reserve),
reserve_failed: Some(reset_match_state_test_reserve_failed),
zero: Some(reset_match_state_test_zero),
}
}
fn reset_match_state_test_params(
strategy: c_int,
search_log: u32,
min_match: u32,
) -> ZSTD_compressionParameters {
ZSTD_compressionParameters {
windowLog: 20,
chainLog: 11,
hashLog: 12,
searchLog: search_log,
minMatch: min_match,
targetLength: 16,
strategy,
}
}
#[test]
fn reset_match_state_orders_cctx_opt_allocations() {
let c_params = reset_match_state_test_params(ZSTD_BTOPT, 5, 3);
let mut context = ResetMatchStateTestContext::default();
let mut hash_log3 = 99;
let mut row_hash_log = 99;
let mut hash_salt = 7;
let mut lazy_skipping = 9;
let mut c_params_out = ZSTD_compressionParameters::default();
let state = reset_match_state_test_state(
&mut context,
c_params,
0,
ZSTD_RUST_PS_DISABLE,
0,
1,
1,
&mut hash_log3,
&mut row_hash_log,
&mut hash_salt,
&mut lazy_skipping,
&mut c_params_out,
);
let result = unsafe { ZSTD_rust_resetMatchState(&state) };
assert_eq!(result, 0);
assert_eq!(hash_log3, 17);
assert_eq!(row_hash_log, 99);
assert_eq!(hash_salt, 7);
assert_eq!(lazy_skipping, 0);
assert_eq!(c_params_out, c_params);
assert_eq!(
context.events,
[
1, 2, 3, 4, 100, 200, 100, 201, 100, 202, 5, 102, 204, 102, 205, 102, 206, 102,
207, 102, 208, 102, 209,
]
);
assert_eq!(
context.reserves,
[
(0, (1 << 12) * size_of::<c_uint>()),
(0, (1 << 11) * size_of::<c_uint>()),
(0, (1 << 17) * size_of::<c_uint>()),
(2, 256 * size_of::<c_uint>()),
(2, 36 * size_of::<c_uint>()),
(2, 53 * size_of::<c_uint>()),
(2, 32 * size_of::<c_uint>()),
(2, 4099 * 2 * size_of::<c_uint>()),
(2, 4099 * 8 * size_of::<c_uint>()),
]
);
assert_eq!(context.pointers.len(), 9);
assert!(context.zeroed.is_empty());
}
#[test]
fn reset_match_state_salts_cctx_row_tables_without_zeroing() {
let c_params = reset_match_state_test_params(ZSTD_GREEDY, 6, 4);
let mut context = ResetMatchStateTestContext::default();
let mut hash_log3 = 99;
let mut row_hash_log = 99;
let mut hash_salt = 7;
let mut lazy_skipping = 9;
let mut c_params_out = ZSTD_compressionParameters::default();
let state = reset_match_state_test_state(
&mut context,
c_params,
0,
ZSTD_RUST_PS_ENABLE,
0,
0,
1,
&mut hash_log3,
&mut row_hash_log,
&mut hash_salt,
&mut lazy_skipping,
&mut c_params_out,
);
let result = unsafe { ZSTD_rust_resetMatchState(&state) };
assert_eq!(result, 0);
assert_eq!(hash_log3, 0);
assert_eq!(row_hash_log, 6);
assert_eq!(hash_salt, 7);
assert_eq!(c_params_out, c_params);
assert_eq!(
context.events,
[3, 4, 100, 200, 100, 201, 100, 202, 5, 101, 203, 6]
);
assert_eq!(
context.reserves,
[
(0, (1 << 12) * size_of::<c_uint>()),
(0, 0),
(0, 0),
(1, 1 << 12),
]
);
assert_eq!(context.pointers.len(), 4);
assert!(context.zeroed.is_empty());
}
#[test]
fn reset_match_state_uses_cdict_row_policy_without_opt_storage() {
let c_params = reset_match_state_test_params(ZSTD_GREEDY, 6, 4);
let mut context = ResetMatchStateTestContext::default();
let mut hash_log3 = 99;
let mut row_hash_log = 99;
let mut hash_salt = 7;
let mut lazy_skipping = 9;
let mut c_params_out = ZSTD_compressionParameters::default();
let state = reset_match_state_test_state(
&mut context,
c_params,
1,
ZSTD_RUST_PS_ENABLE,
1,
0,
0,
&mut hash_log3,
&mut row_hash_log,
&mut hash_salt,
&mut lazy_skipping,
&mut c_params_out,
);
let result = unsafe { ZSTD_rust_resetMatchState(&state) };
assert_eq!(result, 0);
assert_eq!(hash_log3, 0);
assert_eq!(row_hash_log, 6);
assert_eq!(hash_salt, 0);
assert_eq!(c_params_out, c_params);
assert_eq!(
context.events,
[3, 4, 100, 200, 100, 201, 100, 202, 102, 203, 7]
);
assert_eq!(
context.reserves,
[
(0, (1 << 12) * size_of::<c_uint>()),
(0, (1 << 11) * size_of::<c_uint>()),
(0, 0),
(2, 1 << 12),
]
);
assert_eq!(context.pointers.len(), 4);
assert_eq!(context.zeroed.len(), 1);
assert_eq!(context.zeroed[0].1, 1 << 12);
}
#[test]
fn reset_match_state_propagates_table_allocation_failure() {
let c_params = reset_match_state_test_params(ZSTD_GREEDY, 4, 4);
let mut context = ResetMatchStateTestContext {
fail_after: 1,
..ResetMatchStateTestContext::default()
};
let mut hash_log3 = 99;
let mut row_hash_log = 99;
let mut hash_salt = 7;
let mut lazy_skipping = 9;
let mut c_params_out = ZSTD_compressionParameters::default();
let state = reset_match_state_test_state(
&mut context,
c_params,
0,
ZSTD_RUST_PS_DISABLE,
0,
1,
1,
&mut hash_log3,
&mut row_hash_log,
&mut hash_salt,
&mut lazy_skipping,
&mut c_params_out,
);
let result = unsafe { ZSTD_rust_resetMatchState(&state) };
assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation));
assert_eq!(context.events, [1, 2, 3, 4, 100, 200, 100, 201, 100, 202]);
assert_eq!(context.pointers.len(), 3);
assert_eq!(hash_log3, 0);
assert_eq!(lazy_skipping, 0);
assert_eq!(c_params_out, ZSTD_compressionParameters::default());
}
#[test]
fn reference_external_sequences_resets_store_state() {
let mut source = [0u8; 3];