feat(compress): move match-state reset policy into Rust
Move the reset decision tree for match-state tables from C into a Rust-owned callback projection. Rust now decides chain/hash3/row/optimal allocations, reset versus continue behavior, clean versus leave-dirty policy, scalar publication, and allocation-error propagation. C retains the cwksp phases, window invalidation, private pointer assignments, zeroing, and hash-salt implementation behind callbacks, so the private match-state and workspace layouts remain C-owned. Update the migration boundary documentation and add callback-order probes for CCtx optimal storage, CCtx row salting, CDict row zeroing, and table allocation failure. Test Plan: - cargo fmt --manifest-path rust/Cargo.toml -- --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (677 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; make -B -C programs -j1 zstd - ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzzer rounds)
This commit is contained in:
+234
-76
@@ -614,6 +614,65 @@ U64 ZSTD_rust_advanceHashSalt(U64 hashSalt, U64 hashSaltEntropy);
|
||||
int ZSTD_rust_indexTooCloseToMax(size_t nextSrcBaseOffset);
|
||||
int ZSTD_rust_dictTooBig(size_t loadedDictSize);
|
||||
|
||||
/* Match-state reset orchestration lives in Rust. The callbacks keep the
|
||||
* workspace implementation, window representation, and private pointer
|
||||
* fields in C. */
|
||||
typedef void (*ZSTD_rust_resetMatchStateCallback_f)(void* context);
|
||||
typedef void (*ZSTD_rust_resetMatchStateSetPointer_f)(
|
||||
void* context, int pointerKind, void* pointer);
|
||||
typedef void* (*ZSTD_rust_resetMatchStateReserve_f)(
|
||||
void* context, int reserveKind, size_t size);
|
||||
typedef int (*ZSTD_rust_resetMatchStateReserveFailed_f)(void* context);
|
||||
typedef void (*ZSTD_rust_resetMatchStateZero_f)(
|
||||
void* context, void* pointer, size_t size);
|
||||
typedef struct {
|
||||
void* callbackContext;
|
||||
ZSTD_compressionParameters cParams;
|
||||
int dedicatedDictSearch;
|
||||
int useRowMatchFinder;
|
||||
int compResetPolicy;
|
||||
int indexResetPolicy;
|
||||
int resetTarget;
|
||||
unsigned hashLog3Max;
|
||||
size_t litFreqSize;
|
||||
size_t litLengthFreqSize;
|
||||
size_t matchLengthFreqSize;
|
||||
size_t offCodeFreqSize;
|
||||
size_t matchSize;
|
||||
size_t optimalSize;
|
||||
void* hashLog3;
|
||||
void* rowHashLog;
|
||||
void* hashSalt;
|
||||
void* lazySkipping;
|
||||
void* cParamsOut;
|
||||
ZSTD_rust_resetMatchStateSetPointer_f setPointer;
|
||||
ZSTD_rust_resetMatchStateCallback_f windowInit;
|
||||
ZSTD_rust_resetMatchStateCallback_f markTablesDirty;
|
||||
ZSTD_rust_resetMatchStateCallback_f invalidateMatchState;
|
||||
ZSTD_rust_resetMatchStateCallback_f clearTables;
|
||||
ZSTD_rust_resetMatchStateCallback_f cleanTables;
|
||||
ZSTD_rust_resetMatchStateCallback_f advanceHashSalt;
|
||||
ZSTD_rust_resetMatchStateReserve_f reserve;
|
||||
ZSTD_rust_resetMatchStateReserveFailed_f reserveFailed;
|
||||
ZSTD_rust_resetMatchStateZero_f zero;
|
||||
} ZSTD_rust_resetMatchStateState;
|
||||
size_t ZSTD_rust_resetMatchState(const ZSTD_rust_resetMatchStateState* state);
|
||||
typedef char ZSTD_rust_reset_match_state_layout[
|
||||
(offsetof(ZSTD_rust_resetMatchStateState, callbackContext) == 0
|
||||
&& offsetof(ZSTD_rust_resetMatchStateState, cParams) == sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_resetMatchStateState, litFreqSize)
|
||||
> offsetof(ZSTD_rust_resetMatchStateState, hashLog3Max)
|
||||
&& offsetof(ZSTD_rust_resetMatchStateState, hashLog3)
|
||||
== offsetof(ZSTD_rust_resetMatchStateState, optimalSize)
|
||||
+ sizeof(size_t)
|
||||
&& offsetof(ZSTD_rust_resetMatchStateState, setPointer)
|
||||
== offsetof(ZSTD_rust_resetMatchStateState, cParamsOut)
|
||||
+ sizeof(void*)
|
||||
&& sizeof(ZSTD_rust_resetMatchStateState)
|
||||
== offsetof(ZSTD_rust_resetMatchStateState, zero)
|
||||
+ sizeof(void*))
|
||||
? 1 : -1];
|
||||
|
||||
/* The frame-chunk loop is Rust-owned. Its callbacks keep the private
|
||||
* ZSTD_CCtx and match-state layout in C: Rust only drives the block loop and
|
||||
* passes this context back to these C-owned state-preparation/dispatch seams. */
|
||||
@@ -3308,6 +3367,147 @@ static void ZSTD_advanceHashSalt(ZSTD_MatchState_t* ms) {
|
||||
ms->hashSalt = ZSTD_rust_advanceHashSalt(ms->hashSalt, (U64)ms->hashSaltEntropy);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
ZSTD_MatchState_t* ms;
|
||||
ZSTD_cwksp* ws;
|
||||
} ZSTD_rust_resetMatchStateContext;
|
||||
|
||||
enum {
|
||||
ZSTD_RUST_RESET_MATCH_RESERVE_TABLE = 0,
|
||||
ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE = 1,
|
||||
ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED64 = 2
|
||||
};
|
||||
|
||||
enum {
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE = 0,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_CHAIN_TABLE = 1,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE3 = 2,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_TAG_TABLE = 3,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_LIT_FREQ = 4,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_LIT_LENGTH_FREQ = 5,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_MATCH_LENGTH_FREQ = 6,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_OFF_CODE_FREQ = 7,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_MATCH_TABLE = 8,
|
||||
ZSTD_RUST_RESET_MATCH_POINTER_PRICE_TABLE = 9
|
||||
};
|
||||
|
||||
static void ZSTD_rust_resetMatchState_windowInit(void* opaque)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
ZSTD_window_init(&context->ms->window);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetMatchState_markTablesDirty(void* opaque)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
ZSTD_cwksp_mark_tables_dirty(context->ws);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetMatchState_invalidate(void* opaque)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
ZSTD_invalidateMatchState(context->ms);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetMatchState_clearTables(void* opaque)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
ZSTD_cwksp_clear_tables(context->ws);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetMatchState_cleanTables(void* opaque)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
ZSTD_cwksp_clean_tables(context->ws);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetMatchState_advanceHashSalt(void* opaque)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
ZSTD_advanceHashSalt(context->ms);
|
||||
}
|
||||
|
||||
static void* ZSTD_rust_resetMatchState_reserve(
|
||||
void* opaque, int reserveKind, size_t size)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
switch (reserveKind) {
|
||||
case ZSTD_RUST_RESET_MATCH_RESERVE_TABLE:
|
||||
return ZSTD_cwksp_reserve_table(context->ws, size);
|
||||
case ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE:
|
||||
return ZSTD_cwksp_reserve_aligned_init_once(context->ws, size);
|
||||
case ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED64:
|
||||
return ZSTD_cwksp_reserve_aligned64(context->ws, size);
|
||||
default:
|
||||
assert(0);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static int ZSTD_rust_resetMatchState_reserveFailed(void* opaque)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
return ZSTD_cwksp_reserve_failed(context->ws);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetMatchState_setPointer(
|
||||
void* opaque, int pointerKind, void* pointer)
|
||||
{
|
||||
ZSTD_rust_resetMatchStateContext* const context =
|
||||
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||
ZSTD_MatchState_t* const ms = context->ms;
|
||||
switch (pointerKind) {
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE:
|
||||
ms->hashTable = (U32*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_CHAIN_TABLE:
|
||||
ms->chainTable = (U32*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE3:
|
||||
ms->hashTable3 = (U32*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_TAG_TABLE:
|
||||
ms->tagTable = (BYTE*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_LIT_FREQ:
|
||||
ms->opt.litFreq = (unsigned*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_LIT_LENGTH_FREQ:
|
||||
ms->opt.litLengthFreq = (unsigned*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_MATCH_LENGTH_FREQ:
|
||||
ms->opt.matchLengthFreq = (unsigned*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_OFF_CODE_FREQ:
|
||||
ms->opt.offCodeFreq = (unsigned*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_MATCH_TABLE:
|
||||
ms->opt.matchTable = (ZSTD_match_t*)pointer;
|
||||
break;
|
||||
case ZSTD_RUST_RESET_MATCH_POINTER_PRICE_TABLE:
|
||||
ms->opt.priceTable = (ZSTD_optimal_t*)pointer;
|
||||
break;
|
||||
default:
|
||||
assert(0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetMatchState_zero(
|
||||
void* opaque, void* pointer, size_t size)
|
||||
{
|
||||
(void)opaque;
|
||||
ZSTD_memset(pointer, 0, size);
|
||||
}
|
||||
|
||||
static size_t
|
||||
ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
|
||||
ZSTD_cwksp* ws,
|
||||
@@ -3317,82 +3517,40 @@ ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
|
||||
const ZSTD_indexResetPolicy_e forceResetIndex,
|
||||
const ZSTD_resetTarget_e forWho)
|
||||
{
|
||||
/* disable chain table allocation for fast or row-based strategies */
|
||||
size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder,
|
||||
ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict))
|
||||
? ((size_t)1 << cParams->chainLog)
|
||||
: 0;
|
||||
size_t const hSize = ((size_t)1) << cParams->hashLog;
|
||||
U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
|
||||
size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
|
||||
|
||||
DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
|
||||
assert(useRowMatchFinder != ZSTD_ps_auto);
|
||||
if (forceResetIndex == ZSTDirp_reset) {
|
||||
ZSTD_window_init(&ms->window);
|
||||
ZSTD_cwksp_mark_tables_dirty(ws);
|
||||
}
|
||||
|
||||
ms->hashLog3 = hashLog3;
|
||||
ms->lazySkipping = 0;
|
||||
|
||||
ZSTD_invalidateMatchState(ms);
|
||||
|
||||
assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */
|
||||
|
||||
ZSTD_cwksp_clear_tables(ws);
|
||||
|
||||
DEBUGLOG(5, "reserving table space");
|
||||
/* table Space */
|
||||
ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32));
|
||||
ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32));
|
||||
ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32));
|
||||
RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
|
||||
"failed a workspace allocation in ZSTD_reset_matchState");
|
||||
|
||||
DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty);
|
||||
if (crp!=ZSTDcrp_leaveDirty) {
|
||||
/* reset tables only */
|
||||
ZSTD_cwksp_clean_tables(ws);
|
||||
}
|
||||
|
||||
if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
|
||||
/* Row match finder needs an additional table of hashes ("tags") */
|
||||
size_t const tagTableSize = hSize;
|
||||
/* We want to generate a new salt in case we reset a Cctx, but we always want to use
|
||||
* 0 when we reset a Cdict */
|
||||
if(forWho == ZSTD_resetTarget_CCtx) {
|
||||
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize);
|
||||
ZSTD_advanceHashSalt(ms);
|
||||
} else {
|
||||
/* When we are not salting we want to always memset the memory */
|
||||
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize);
|
||||
ZSTD_memset(ms->tagTable, 0, tagTableSize);
|
||||
ms->hashSalt = 0;
|
||||
}
|
||||
{ /* Switch to 32-entry rows if searchLog is 5 (or more) */
|
||||
U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
|
||||
assert(cParams->hashLog >= rowLog);
|
||||
ms->rowHashLog = cParams->hashLog - rowLog;
|
||||
}
|
||||
}
|
||||
|
||||
/* opt parser space */
|
||||
if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
|
||||
DEBUGLOG(4, "reserving optimal parser space");
|
||||
ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<<Litbits) * sizeof(unsigned));
|
||||
ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned));
|
||||
ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned));
|
||||
ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned));
|
||||
ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t));
|
||||
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t));
|
||||
}
|
||||
|
||||
ms->cParams = *cParams;
|
||||
|
||||
RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
|
||||
"failed a workspace allocation in ZSTD_reset_matchState");
|
||||
return 0;
|
||||
ZSTD_rust_resetMatchStateContext context;
|
||||
ZSTD_rust_resetMatchStateState state;
|
||||
context.ms = ms;
|
||||
context.ws = ws;
|
||||
state.callbackContext = &context;
|
||||
state.cParams = *cParams;
|
||||
state.dedicatedDictSearch = ms->dedicatedDictSearch;
|
||||
state.useRowMatchFinder = (int)useRowMatchFinder;
|
||||
state.compResetPolicy = (int)crp;
|
||||
state.indexResetPolicy = (int)forceResetIndex;
|
||||
state.resetTarget = (int)forWho;
|
||||
state.hashLog3Max = ZSTD_HASHLOG3_MAX;
|
||||
state.litFreqSize = (1 << Litbits) * sizeof(unsigned);
|
||||
state.litLengthFreqSize = (MaxLL + 1) * sizeof(unsigned);
|
||||
state.matchLengthFreqSize = (MaxML + 1) * sizeof(unsigned);
|
||||
state.offCodeFreqSize = (MaxOff + 1) * sizeof(unsigned);
|
||||
state.matchSize = ZSTD_OPT_SIZE * sizeof(ZSTD_match_t);
|
||||
state.optimalSize = ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t);
|
||||
state.hashLog3 = &ms->hashLog3;
|
||||
state.rowHashLog = &ms->rowHashLog;
|
||||
state.hashSalt = &ms->hashSalt;
|
||||
state.lazySkipping = &ms->lazySkipping;
|
||||
state.cParamsOut = &ms->cParams;
|
||||
state.setPointer = ZSTD_rust_resetMatchState_setPointer;
|
||||
state.windowInit = ZSTD_rust_resetMatchState_windowInit;
|
||||
state.markTablesDirty = ZSTD_rust_resetMatchState_markTablesDirty;
|
||||
state.invalidateMatchState = ZSTD_rust_resetMatchState_invalidate;
|
||||
state.clearTables = ZSTD_rust_resetMatchState_clearTables;
|
||||
state.cleanTables = ZSTD_rust_resetMatchState_cleanTables;
|
||||
state.advanceHashSalt = ZSTD_rust_resetMatchState_advanceHashSalt;
|
||||
state.reserve = ZSTD_rust_resetMatchState_reserve;
|
||||
state.reserveFailed = ZSTD_rust_resetMatchState_reserveFailed;
|
||||
state.zero = ZSTD_rust_resetMatchState_zero;
|
||||
return ZSTD_rust_resetMatchState(&state);
|
||||
}
|
||||
|
||||
/* ZSTD_indexTooCloseToMax() :
|
||||
|
||||
Reference in New Issue
Block a user